pixeltable 0.3.2__py3-none-any.whl → 0.3.3__py3-none-any.whl

pixeltable/exec/expr_eval/globals.py

@@ -1,16 +1,18 @@
+from __future__ import annotations
+
 import abc
 import asyncio
 from dataclasses import dataclass
 from types import TracebackType
-from typing import Any, Protocol, Optional
+from typing import Any, Optional, Protocol
 
-from pixeltable import exprs
-from pixeltable import func
+from pixeltable import exprs, func
 
 
 @dataclass
 class FnCallArgs:
     """Container for everything needed to execute a FunctionCall against one or more DataRows"""
+
     fn_call: exprs.FunctionCall
     rows: list[exprs.DataRow]
     # single call
@@ -37,16 +39,36 @@ class FnCallArgs:
 
 class Scheduler(abc.ABC):
     """
-    Base class for schedulers. A scheduler executes FunctionCalls against a limited resource pool.
+    Base class for queueing schedulers. A scheduler executes FunctionCalls against a limited resource pool.
 
     Expected behavior:
     - all created tasks must be recorded in dispatcher.tasks
     - schedulers are responsible for aborting execution when a) the task is cancelled or b) when an exception occurred
       elsewhere (indicated by dispatcher.exc_event)
     """
-    @abc.abstractmethod
+
+    @dataclass(frozen=True)
+    class QueueItem:
+        """Container of work items for queueing schedulers"""
+
+        request: FnCallArgs
+        num_retries: int
+
+        def __lt__(self, other: Scheduler.QueueItem) -> bool:
+            # prioritize by number of retries (more retries = higher priority)
+            return self.num_retries > other.num_retries
+
+    resource_pool: str
+    queue: asyncio.PriorityQueue[QueueItem]  # prioritizes retries
+    dispatcher: Dispatcher
+
+    def __init__(self, resource_pool: str, dispatcher: Dispatcher):
+        self.resource_pool = resource_pool
+        self.queue = asyncio.PriorityQueue()
+        self.dispatcher = dispatcher
+
     def submit(self, item: FnCallArgs) -> None:
-        pass
+        self.queue.put_nowait(self.QueueItem(item, 0))
 
     @classmethod
     @abc.abstractmethod
@@ -63,6 +85,7 @@ class Dispatcher(Protocol):
     Exceptions: evaluators/schedulers need to check exc_event prior to starting long-running (non-interruptible)
         computations
     """
+
     row_builder: exprs.RowBuilder
     exc_event: asyncio.Event
     schedulers: dict[str, Scheduler]  # key: resource pool id
@@ -90,6 +113,7 @@ class Evaluator(abc.ABC):
     - evaluators are responsible for aborting execution when a) the task is cancelled or b) when an exception occurred
       elsewhere (indicated by dispatcher.exc_event)
     """
+
     dispatcher: Dispatcher
     is_closed: bool
 
@@ -110,4 +134,3 @@ class Evaluator(abc.ABC):
         """Indicates that there may not be any more rows getting scheduled"""
         self.is_closed = True
         self._close()
-

pixeltable/exec/expr_eval/row_buffer.py

@@ -62,15 +62,14 @@ class RowBuffer:
             return []
         rows: list[exprs.DataRow]
         if self.head_idx + n <= self.size:
-            rows = self.buffer[self.head_idx:self.head_idx + n].tolist()
-            self.buffer[self.head_idx:self.head_idx + n] = None
+            rows = self.buffer[self.head_idx : self.head_idx + n].tolist()
+            self.buffer[self.head_idx : self.head_idx + n] = None
         else:
-            rows = np.concatenate([self.buffer[self.head_idx:], self.buffer[:self.head_idx + n - self.size]]).tolist()
-            self.buffer[self.head_idx:] = None
-            self.buffer[:self.head_idx + n - self.size] = None
+            rows = np.concatenate([self.buffer[self.head_idx :], self.buffer[: self.head_idx + n - self.size]]).tolist()
+            self.buffer[self.head_idx :] = None
+            self.buffer[: self.head_idx + n - self.size] = None
         self.head_pos += n
         self.head_idx = (self.head_idx + n) % self.size
         self.num_rows -= n
         self.num_ready -= n
         return rows
-

pixeltable/exec/expr_eval/schedulers.py

@@ -5,12 +5,12 @@ import datetime
 import inspect
 import logging
 import sys
-from dataclasses import dataclass
-from typing import Optional, Awaitable, Collection
+import time
+from typing import Awaitable, Collection, Optional
 
-from pixeltable import env
-from pixeltable import func
-from .globals import Scheduler, FnCallArgs, Dispatcher
+from pixeltable import env, func
+
+from .globals import Dispatcher, FnCallArgs, Scheduler
 
 _logger = logging.getLogger('pixeltable')
 
@@ -29,19 +29,7 @@ class RateLimitsScheduler(Scheduler):
     TODO:
     - limit the number of in-flight requests based on the open file limit
     """
-    @dataclass(frozen=True)
-    class QueueItem:
-        request: FnCallArgs
-        num_retries: int
-
-        def __lt__(self, other: RateLimitsScheduler.QueueItem) -> bool:
-            # prioritize by number of retries
-            return self.num_retries > other.num_retries
-
-    resource_pool: str
-    queue: asyncio.PriorityQueue[QueueItem]  # prioritizes retries
-    loop_task: asyncio.Task
-    dispatcher: Dispatcher
+
     get_request_resources_param_names: list[str]  # names of parameters of RateLimitsInfo.get_request_resources()
 
     # scheduling-related state
@@ -58,11 +46,9 @@ class RateLimitsScheduler(Scheduler):
     MAX_RETRIES = 10
 
     def __init__(self, resource_pool: str, dispatcher: Dispatcher):
-        self.resource_pool = resource_pool
-        self.queue = asyncio.PriorityQueue()
-        self.dispatcher = dispatcher
-        self.loop_task = asyncio.create_task(self._main_loop())
-        self.dispatcher.register_task(self.loop_task)
+        super().__init__(resource_pool, dispatcher)
+        loop_task = asyncio.create_task(self._main_loop())
+        self.dispatcher.register_task(loop_task)
         self.pool_info = None  # initialized in _main_loop by the first request
         self.est_usage = {}
         self.num_in_flight = 0
@@ -104,6 +90,7 @@ class RateLimitsScheduler(Scheduler):
                 # wait for a single request to get rate limits
                 _logger.debug(f'initializing rate limits for {self.resource_pool}')
                 await self._exec(item.request, item.num_retries, is_task=False)
+                _logger.debug(f'initialized rate limits for {self.resource_pool}')
                 item = None
                 # if this was the first request, it created the pool_info
                 if self.pool_info is None:
@@ -111,6 +98,7 @@ class RateLimitsScheduler(Scheduler):
                 continue
 
             # check rate limits
+            _logger.debug(f'checking rate limits for {self.resource_pool}')
             request_resources = self._get_request_resources(item.request)
             limits_info = self._check_resource_limits(request_resources)
             aws: list[Awaitable[None]] = []
@@ -169,7 +157,6 @@ class RateLimitsScheduler(Scheduler):
             constant_kwargs, batch_kwargs = request.pxt_fn.create_batch_kwargs(batch_kwargs)
             return self.pool_info.get_request_resources(**constant_kwargs, **batch_kwargs)
 
-
     def _check_resource_limits(self, request_resources: dict[str, int]) -> Optional[env.RateLimitInfo]:
         """Returns the most depleted resource, relative to its limit, or None if all resources are within limits"""
         candidates: list[tuple[env.RateLimitInfo, float]] = []  # (info, relative usage)
@@ -191,7 +178,9 @@ class RateLimitsScheduler(Scheduler):
             start_ts = datetime.datetime.now(tz=datetime.timezone.utc)
             pxt_fn = request.fn_call.fn
             assert isinstance(pxt_fn, func.CallableFunction)
-            _logger.debug(f'scheduler {self.resource_pool}: start evaluating slot {request.fn_call.slot_idx}, batch_size={len(request.rows)}')
+            _logger.debug(
+                f'scheduler {self.resource_pool}: start evaluating slot {request.fn_call.slot_idx}, batch_size={len(request.rows)}'
+            )
             self.total_requests += 1
             if request.is_batched:
                 batch_result = await pxt_fn.aexec_batch(*request.batch_args, **request.batch_kwargs)
@@ -202,7 +191,9 @@ class RateLimitsScheduler(Scheduler):
                 result = await pxt_fn.aexec(*request.args, **request.kwargs)
                 request.row[request.fn_call.slot_idx] = result
             end_ts = datetime.datetime.now(tz=datetime.timezone.utc)
-            _logger.debug(f'scheduler {self.resource_pool}: evaluated slot {request.fn_call.slot_idx} in {end_ts - start_ts}, batch_size={len(request.rows)}')
+            _logger.debug(
+                f'scheduler {self.resource_pool}: evaluated slot {request.fn_call.slot_idx} in {end_ts - start_ts}, batch_size={len(request.rows)}'
+            )
 
             # purge accumulated usage estimate, now that we have a new report
             self.est_usage = {r: 0 for r in self._resources}
@@ -210,10 +201,11 @@ class RateLimitsScheduler(Scheduler):
             self.dispatcher.dispatch(request.rows)
         except Exception as exc:
             _logger.debug(f'scheduler {self.resource_pool}: exception in slot {request.fn_call.slot_idx}: {exc}')
-            if  self.pool_info is None:
+            if self.pool_info is None:
                 # our pool info should be available at this point
                 self._set_pool_info()
-            if num_retries < self.MAX_RETRIES and self.pool_info is not None:
+            assert self.pool_info is not None
+            if num_retries < self.MAX_RETRIES:
                 retry_delay = self.pool_info.get_retry_delay(exc)
                 if retry_delay is not None:
                     self.total_retried += 1
@@ -229,12 +221,140 @@ class RateLimitsScheduler(Scheduler):
                 row.set_exc(request.fn_call.slot_idx, exc)
             self.dispatcher.dispatch_exc(request.rows, request.fn_call.slot_idx, exc_tb)
         finally:
-            _logger.debug(
-                f'Scheduler stats: #requests={self.total_requests}, #retried={self.total_retried}')
+            _logger.debug(f'Scheduler stats: #requests={self.total_requests}, #retried={self.total_retried}')
             if is_task:
                 self.num_in_flight -= 1
                 self.request_completed.set()
 
 
+class RequestRateScheduler(Scheduler):
+    """
+    Scheduler for FunctionCalls with a fixed request rate limit and no runtime resource usage reports.
+
+    Rate limits are supplied in the config, in one of two ways:
+    - resource_pool='request-rate:<endpoint>':
+      * a single rate limit for all calls against that endpoint
+      * in the config: section '<endpoint>', key 'rate_limit'
+    - resource_pool='request-rate:<endpoint>:<model>':
+        * a single rate limit for all calls against that model
+        * in the config: section '<endpoint>.rate_limits', key '<model>'
+    - if no rate limit is found in the config, uses a default of 600 RPM
+
+    TODO:
+    - adaptive rate limiting based on 429 errors
+    """
+
+    secs_per_request: float  # inverted rate limit
+    num_in_flight: int
+    total_requests: int
+    total_retried: int
+
+    TIME_FORMAT = '%H:%M.%S %f'
+    MAX_RETRIES = 10
+    DEFAULT_RATE_LIMIT = 600  # requests per minute
+
+    def __init__(self, resource_pool: str, dispatcher: Dispatcher):
+        super().__init__(resource_pool, dispatcher)
+        loop_task = asyncio.create_task(self._main_loop())
+        self.dispatcher.register_task(loop_task)
+        self.num_in_flight = 0
+        self.total_requests = 0
+        self.total_retried = 0
+
+        # try to get the rate limit from the config
+        elems = resource_pool.split(':')
+        section: str
+        key: str
+        if len(elems) == 2:
+            # resource_pool: request-rate:endpoint
+            _, endpoint = elems
+            section = endpoint
+            key = 'rate_limit'
+        else:
+            # resource_pool: request-rate:endpoint:model
+            assert len(elems) == 3
+            _, endpoint, model = elems
+            section = f'{endpoint}.rate_limits'
+            key = model
+        requests_per_min = env.Env.get().config.get_int_value(key, section=section)
+        requests_per_min = requests_per_min or self.DEFAULT_RATE_LIMIT
+        self.secs_per_request = 1 / (requests_per_min / 60)
+
+    @classmethod
+    def matches(cls, resource_pool: str) -> bool:
+        return resource_pool.startswith('request-rate:')
+
+    async def _main_loop(self) -> None:
+        last_request_ts = 0.0
+        while True:
+            item = await self.queue.get()
+            if item.num_retries > 0:
+                self.total_retried += 1
+            now = time.monotonic()
+            if now - last_request_ts < self.secs_per_request:
+                wait_duration = self.secs_per_request - (now - last_request_ts)
+                _logger.debug(f'waiting for {wait_duration} for {self.resource_pool}')
+                await asyncio.sleep(wait_duration)
+
+            last_request_ts = time.monotonic()
+            if item.num_retries > 0:
+                # the last request encountered some problem: retry it synchronously, to wait for the problem to pass
+                _logger.debug(f'retrying request for {self.resource_pool}: #retries={item.num_retries}')
+                await self._exec(item.request, item.num_retries, is_task=False)
+                _logger.debug(f'retried request for {self.resource_pool}: #retries={item.num_retries}')
+            else:
+                _logger.debug(f'creating task for {self.resource_pool}')
+                self.num_in_flight += 1
+                task = asyncio.create_task(self._exec(item.request, item.num_retries, is_task=True))
+                self.dispatcher.register_task(task)
+
+    async def _exec(self, request: FnCallArgs, num_retries: int, is_task: bool) -> None:
+        assert all(not row.has_val[request.fn_call.slot_idx] for row in request.rows)
+        assert all(not row.has_exc(request.fn_call.slot_idx) for row in request.rows)
+
+        try:
+            start_ts = datetime.datetime.now(tz=datetime.timezone.utc)
+            pxt_fn = request.fn_call.fn
+            assert isinstance(pxt_fn, func.CallableFunction)
+            _logger.debug(
+                f'scheduler {self.resource_pool}: start evaluating slot {request.fn_call.slot_idx}, batch_size={len(request.rows)}'
+            )
+            self.total_requests += 1
+            if request.is_batched:
+                batch_result = await pxt_fn.aexec_batch(*request.batch_args, **request.batch_kwargs)
+                assert len(batch_result) == len(request.rows)
+                for row, result in zip(request.rows, batch_result):
+                    row[request.fn_call.slot_idx] = result
+            else:
+                result = await pxt_fn.aexec(*request.args, **request.kwargs)
+                request.row[request.fn_call.slot_idx] = result
+            end_ts = datetime.datetime.now(tz=datetime.timezone.utc)
+            _logger.debug(
+                f'scheduler {self.resource_pool}: evaluated slot {request.fn_call.slot_idx} in {end_ts - start_ts}, batch_size={len(request.rows)}'
+            )
+            self.dispatcher.dispatch(request.rows)
+
+        except Exception as exc:
+            # TODO: which exception can be retried?
+            _logger.debug(f'exception for {self.resource_pool}: {exc}')
+            status = getattr(exc, 'status', None)
+            _logger.debug(f'type={type(exc)} has_status={hasattr(exc, "status")} status={status}')
+            if num_retries < self.MAX_RETRIES:
+                self.queue.put_nowait(self.QueueItem(request, num_retries + 1))
+                return
+
+            # record the exception
+            _, _, exc_tb = sys.exc_info()
+            for row in request.rows:
+                row.set_exc(request.fn_call.slot_idx, exc)
+            self.dispatcher.dispatch_exc(request.rows, request.fn_call.slot_idx, exc_tb)
+        finally:
+            _logger.debug(
+                f'Scheduler stats: #in-flight={self.num_in_flight} #requests={self.total_requests}, #retried={self.total_retried}'
+            )
+            if is_task:
+                self.num_in_flight -= 1
+
+
 # all concrete Scheduler subclasses that implement matches()
-SCHEDULERS = [RateLimitsScheduler]
+SCHEDULERS = [RateLimitsScheduler, RequestRateScheduler]

pixeltable/exec/in_memory_data_node.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, Iterator, Optional, AsyncIterator
+from typing import Any, AsyncIterator, Iterator, Optional
 
 import pixeltable.catalog as catalog
 import pixeltable.exprs as exprs
@@ -10,6 +10,7 @@ from .exec_node import ExecNode
 
 _logger = logging.getLogger('pixeltable')
 
+
 class InMemoryDataNode(ExecNode):
     """
     Outputs in-memory data as a DataRowBatch of a particular table.
@@ -18,6 +19,7 @@ class InMemoryDataNode(ExecNode):
     - with the values provided in the input rows
     - if an input row doesn't provide a value, sets the slot to the column default
     """
+
     tbl: catalog.TableVersion
     input_rows: list[dict[str, Any]]
     start_row_id: int
@@ -27,8 +29,7 @@ class InMemoryDataNode(ExecNode):
     output_exprs: list[exprs.ColumnRef]
 
     def __init__(
-        self, tbl: catalog.TableVersion, rows: list[dict[str, Any]],
-        row_builder: exprs.RowBuilder, start_row_id: int,
+        self, tbl: catalog.TableVersion, rows: list[dict[str, Any]], row_builder: exprs.RowBuilder, start_row_id: int
     ):
         # we materialize the input slots
         output_exprs = list(row_builder.input_exprs)
@@ -43,11 +44,11 @@ class InMemoryDataNode(ExecNode):
         """Create row batch and populate with self.input_rows"""
         user_cols_by_name = {
             col_ref.col.name: exprs.ColumnSlotIdx(col_ref.col, col_ref.slot_idx)
-            for col_ref in self.output_exprs if col_ref.col.name is not None
+            for col_ref in self.output_exprs
+            if col_ref.col.name is not None
         }
         output_cols_by_idx = {
-            col_ref.slot_idx: exprs.ColumnSlotIdx(col_ref.col, col_ref.slot_idx)
-            for col_ref in self.output_exprs
+            col_ref.slot_idx: exprs.ColumnSlotIdx(col_ref.col, col_ref.slot_idx) for col_ref in self.output_exprs
         }
         output_slot_idxs = {e.slot_idx for e in self.output_exprs}
 
@@ -68,7 +69,7 @@ class InMemoryDataNode(ExecNode):
                 input_slot_idxs.add(col_info.slot_idx)
 
             # set the remaining output slots to their default values (presently None)
-            missing_slot_idxs =  output_slot_idxs - input_slot_idxs
+            missing_slot_idxs = output_slot_idxs - input_slot_idxs
             for slot_idx in missing_slot_idxs:
                 col_info = output_cols_by_idx.get(slot_idx)
                 assert col_info is not None

pixeltable/exec/row_update_node.py

@@ -4,11 +4,13 @@ from typing import Any, AsyncIterator
 import pixeltable.catalog as catalog
 import pixeltable.exprs as exprs
 from pixeltable.utils.media_store import MediaStore
+
 from .data_row_batch import DataRowBatch
 from .exec_node import ExecNode
 
 _logger = logging.getLogger('pixeltable')
 
+
 class RowUpdateNode(ExecNode):
     """
     Update individual rows in the input batches, identified by key columns.
@@ -17,9 +19,15 @@ class RowUpdateNode(ExecNode):
     The node assumes that all update dicts contain the same keys, and it populates the slots of the columns present in
     the update list.
     """
+
     def __init__(
-            self, tbl: catalog.TableVersionPath, key_vals_batch: list[tuple], is_rowid_key: bool,
-            col_vals_batch: list[dict[catalog.Column, Any]], row_builder: exprs.RowBuilder, input: ExecNode,
+        self,
+        tbl: catalog.TableVersionPath,
+        key_vals_batch: list[tuple],
+        is_rowid_key: bool,
+        col_vals_batch: list[dict[catalog.Column, Any]],
+        row_builder: exprs.RowBuilder,
+        input: ExecNode,
     ):
         super().__init__(row_builder, [], [], input)
         self.updates = {key_vals: col_vals for key_vals, col_vals in zip(key_vals_batch, col_vals_batch)}
@@ -28,7 +36,8 @@ class RowUpdateNode(ExecNode):
         # retrieve ColumnRefs from the RowBuilder (has slot_idx set)
         all_col_slot_idxs = {
             col_ref.col: col_ref.slot_idx
-            for col_ref in row_builder.unique_exprs if isinstance(col_ref, exprs.ColumnRef)
+            for col_ref in row_builder.unique_exprs
+            if isinstance(col_ref, exprs.ColumnRef)
         }
         self.col_slot_idxs = {col: all_col_slot_idxs[col] for col in col_vals_batch[0].keys()}
         self.key_slot_idxs = {col: all_col_slot_idxs[col] for col in tbl.tbl_version.primary_key_columns()}
@@ -37,8 +46,9 @@ class RowUpdateNode(ExecNode):
     async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
         async for batch in self.input:
             for row in batch:
-                key_vals = row.rowid if self.is_rowid_key else \
-                    tuple(row[slot_idx] for slot_idx in self.key_slot_idxs.values())
+                key_vals = (
+                    row.rowid if self.is_rowid_key else tuple(row[slot_idx] for slot_idx in self.key_slot_idxs.values())
+                )
                 if key_vals not in self.updates:
                     continue
                 self.matched_key_vals.add(key_vals)

pixeltable/exec/sql_node.py

@@ -1,13 +1,14 @@
 import logging
 import warnings
 from decimal import Decimal
-from typing import Iterable, Iterator, NamedTuple, Optional, TYPE_CHECKING, Sequence, AsyncIterator
+from typing import TYPE_CHECKING, AsyncIterator, Iterable, Iterator, NamedTuple, Optional, Sequence
 from uuid import UUID
 
 import sqlalchemy as sql
 
 import pixeltable.catalog as catalog
 import pixeltable.exprs as exprs
+
 from .data_row_batch import DataRowBatch
 from .exec_node import ExecNode
 
@@ -53,10 +54,12 @@ def combine_order_by_clauses(clauses: Iterable[OrderByClause]) -> Optional[Order
 
 
 def print_order_by_clause(clause: OrderByClause) -> str:
-    return ', '.join([
-        f'({item.expr}{", asc=True" if item.asc is True else ""}{", asc=False" if item.asc is False else ""})'
-        for item in clause
-    ])
+    return ', '.join(
+        [
+            f'({item.expr}{", asc=True" if item.asc is True else ""}{", asc=False" if item.asc is False else ""})'
+            for item in clause
+        ]
+    )
 
 
 class SqlNode(ExecNode):
@@ -82,8 +85,12 @@ class SqlNode(ExecNode):
     limit: Optional[int]
 
     def __init__(
-            self, tbl: Optional[catalog.TableVersionPath], row_builder: exprs.RowBuilder,
-            select_list: Iterable[exprs.Expr], sql_elements: exprs.SqlElementCache, set_pk: bool = False
+        self,
+        tbl: Optional[catalog.TableVersionPath],
+        row_builder: exprs.RowBuilder,
+        select_list: Iterable[exprs.Expr],
+        sql_elements: exprs.SqlElementCache,
+        set_pk: bool = False,
     ):
         """
         If row_builder contains references to unstored iter columns, expands the select list to include their
@@ -186,8 +193,11 @@ class SqlNode(ExecNode):
 
     @classmethod
     def create_from_clause(
-            cls, tbl: catalog.TableVersionPath, stmt: sql.Select, refd_tbl_ids: Optional[set[UUID]] = None,
-            exact_version_only: Optional[set[UUID]] = None
+        cls,
+        tbl: catalog.TableVersionPath,
+        stmt: sql.Select,
+        refd_tbl_ids: Optional[set[UUID]] = None,
+        exact_version_only: Optional[set[UUID]] = None,
     ) -> sql.Select:
         """Add From clause to stmt for tables/views referenced by materialized_exprs
         Args:
@@ -220,15 +230,14 @@ class SqlNode(ExecNode):
                 # join tbl to prev_tbl on prev_tbl's rowid cols
                 prev_tbl_rowid_cols = prev_tbl.store_tbl.rowid_columns()
                 tbl_rowid_cols = tbl.store_tbl.rowid_columns()
-                rowid_clauses = \
-                    [c1 == c2 for c1, c2 in zip(prev_tbl_rowid_cols, tbl_rowid_cols[:len(prev_tbl_rowid_cols)])]
+                rowid_clauses = [
+                    c1 == c2 for c1, c2 in zip(prev_tbl_rowid_cols, tbl_rowid_cols[: len(prev_tbl_rowid_cols)])
+                ]
                 stmt = stmt.join(tbl.store_tbl.sa_tbl, sql.and_(*rowid_clauses))
             if tbl.id in exact_version_only:
                 stmt = stmt.where(tbl.store_tbl.v_min_col == tbl.version)
             else:
-                stmt = stmt \
-                    .where(tbl.store_tbl.v_min_col <= tbl.version) \
-                    .where(tbl.store_tbl.v_max_col > tbl.version)
+                stmt = stmt.where(tbl.store_tbl.v_min_col <= tbl.version).where(tbl.store_tbl.v_max_col > tbl.version)
             prev_tbl = tbl
         return stmt
 
@@ -291,7 +300,7 @@ class SqlNode(ExecNode):
 
             # populate output_row
             if self.num_pk_cols > 0:
-                output_row.set_pk(tuple(sql_row[-self.num_pk_cols:]))
+                output_row.set_pk(tuple(sql_row[-self.num_pk_cols :]))
             # copy the output of the SQL query into the output row
             for i, e in enumerate(self.select_list):
                 slot_idx = e.slot_idx
@@ -341,12 +350,16 @@ class SqlScanNode(SqlNode):
 
     Supports filtering and ordering.
     """
+
     exact_version_only: list[catalog.TableVersion]
 
     def __init__(
-        self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
+        self,
+        tbl: catalog.TableVersionPath,
+        row_builder: exprs.RowBuilder,
         select_list: Iterable[exprs.Expr],
-        set_pk: bool = False, exact_version_only: Optional[list[catalog.TableVersion]] = None
+        set_pk: bool = False,
+        exact_version_only: Optional[list[catalog.TableVersion]] = None,
     ):
         """
         Args:
@@ -367,7 +380,8 @@ class SqlScanNode(SqlNode):
         where_clause_tbl_ids = self.where_clause.tbl_ids() if self.where_clause is not None else set()
         refd_tbl_ids = exprs.Expr.all_tbl_ids(self.select_list) | where_clause_tbl_ids | self._ordering_tbl_ids()
         stmt = self.create_from_clause(
-            self.tbl, stmt, refd_tbl_ids, exact_version_only={t.id for t in self.exact_version_only})
+            self.tbl, stmt, refd_tbl_ids, exact_version_only={t.id for t in self.exact_version_only}
+        )
         return stmt
 
 
@@ -377,8 +391,12 @@ class SqlLookupNode(SqlNode):
     """
 
     def __init__(
-        self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
-        select_list: Iterable[exprs.Expr], sa_key_cols: list[sql.Column], key_vals: list[tuple],
+        self,
+        tbl: catalog.TableVersionPath,
+        row_builder: exprs.RowBuilder,
+        select_list: Iterable[exprs.Expr],
+        sa_key_cols: list[sql.Column],
+        key_vals: list[tuple],
     ):
         """
         Args:
@@ -406,11 +424,13 @@ class SqlAggregationNode(SqlNode):
     group_by_items: Optional[list[exprs.Expr]]
 
     def __init__(
-        self, row_builder: exprs.RowBuilder,
+        self,
+        row_builder: exprs.RowBuilder,
         input: SqlNode,
         select_list: Iterable[exprs.Expr],
         group_by_items: Optional[list[exprs.Expr]] = None,
-        limit: Optional[int] = None, exact_version_only: Optional[list[catalog.TableVersion]] = None
+        limit: Optional[int] = None,
+        exact_version_only: Optional[list[catalog.TableVersion]] = None,
     ):
         """
         Args:
@@ -436,12 +456,16 @@ class SqlJoinNode(SqlNode):
     """
     Materializes data from the store via a Select ... From ... that contains joins
     """
+
     input_ctes: list[sql.CTE]
     join_clauses: list['pixeltable.plan.JoinClause']
 
     def __init__(
-        self, row_builder: exprs.RowBuilder,
-        inputs: Sequence[SqlNode], join_clauses: list['pixeltable.plan.JoinClause'], select_list: Iterable[exprs.Expr]
+        self,
+        row_builder: exprs.RowBuilder,
+        inputs: Sequence[SqlNode],
+        join_clauses: list['pixeltable.plan.JoinClause'],
+        select_list: Iterable[exprs.Expr],
     ):
         assert len(inputs) > 1
         assert len(inputs) == len(join_clauses) + 1
@@ -456,16 +480,21 @@ class SqlJoinNode(SqlNode):
 
     def _create_stmt(self) -> sql.Select:
         from pixeltable import plan
+
         stmt = super()._create_stmt()
         stmt = stmt.select_from(self.input_ctes[0])
         for i in range(len(self.join_clauses)):
             join_clause = self.join_clauses[i]
             on_clause = (
-                self.sql_elements.get(join_clause.join_predicate) if join_clause.join_type != plan.JoinType.CROSS
+                self.sql_elements.get(join_clause.join_predicate)
+                if join_clause.join_type != plan.JoinType.CROSS
                 else sql.sql.expression.literal(True)
             )
             is_outer = join_clause.join_type == plan.JoinType.LEFT or join_clause.join_type == plan.JoinType.FULL_OUTER
             stmt = stmt.join(
-                self.input_ctes[i + 1], onclause=on_clause, isouter=is_outer,
-                full=join_clause == plan.JoinType.FULL_OUTER)
+                self.input_ctes[i + 1],
+                onclause=on_clause,
+                isouter=is_outer,
+                full=join_clause == plan.JoinType.FULL_OUTER,
+            )
         return stmt