pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/store.py

@@ -2,23 +2,22 @@ from __future__ import annotations
 
 import abc
 import logging
-import os
 import sys
-import urllib.parse
-import urllib.request
+import time
 import warnings
-from typing import Any, Iterable, Iterator, Literal, Optional, Union
+from typing import Any, Iterable, Iterator
 
 import more_itertools
+import psycopg
 import sqlalchemy as sql
 from tqdm import TqdmWarning, tqdm
 
-from pixeltable import catalog, exceptions as excs, exprs
+from pixeltable import catalog, exceptions as excs
+from pixeltable.catalog.update_status import RowCountStats
 from pixeltable.env import Env
 from pixeltable.exec import ExecNode
 from pixeltable.metadata import schema
 from pixeltable.utils.exception_handler import run_cleanup
-from pixeltable.utils.media_store import MediaStore
 from pixeltable.utils.sql import log_explain, log_stmt
 
 _logger = logging.getLogger('pixeltable')
@@ -35,13 +34,16 @@ class StoreBase:
 
     tbl_version: catalog.TableVersionHandle
     sa_md: sql.MetaData
-    sa_tbl: Optional[sql.Table]
+    sa_tbl: sql.Table | None
     _pk_cols: list[sql.Column]
     v_min_col: sql.Column
     v_max_col: sql.Column
-    base: Optional[StoreBase]
+    base: StoreBase | None
 
-    __INSERT_BATCH_SIZE = 1000
+    # In my cursory experiments this was the optimal batch size: it was an improvement over 5_000 and there was no real
+    # benefit to going higher.
+    # TODO: Perform more rigorous experiments with different table structures and OS environments to refine this.
+    __INSERT_BATCH_SIZE = 10_000
 
     def __init__(self, tbl_version: catalog.TableVersion):
         self.tbl_version = catalog.TableVersionHandle(
@@ -78,20 +80,20 @@ class StoreBase:
         self._pk_cols = [*rowid_cols, self.v_min_col]
         return [*rowid_cols, self.v_min_col, self.v_max_col]
 
-    def create_sa_tbl(self, tbl_version: Optional[catalog.TableVersion] = None) -> None:
+    def create_sa_tbl(self, tbl_version: catalog.TableVersion | None = None) -> None:
         """Create self.sa_tbl from self.tbl_version."""
         if tbl_version is None:
             tbl_version = self.tbl_version.get()
         system_cols = self._create_system_columns()
         all_cols = system_cols.copy()
+        # we captured all columns, including dropped ones: they're still part of the physical table
         for col in [c for c in tbl_version.cols if c.is_stored]:
             # re-create sql.Column for each column, regardless of whether it already has sa_col set: it was bound
             # to the last sql.Table version we created and cannot be reused
             col.create_sa_cols()
             all_cols.append(col.sa_col)
-            if col.records_errors:
-                all_cols.append(col.sa_errormsg_col)
-                all_cols.append(col.sa_errortype_col)
+            if col.stores_cellmd:
+                all_cols.append(col.sa_cellmd_col)
 
         if self.sa_tbl is not None:
             # if we're called in response to a schema change, we need to remove the old table first
@@ -122,51 +124,6 @@ class StoreBase:
     def _storage_name(self) -> str:
         """Return the name of the data store table"""
 
-    def _move_tmp_media_file(self, file_url: Optional[str], col: catalog.Column, v_min: int) -> str:
-        """Move tmp media file with given url to Env.media_dir and return new url, or given url if not a tmp_dir file"""
-        pxt_tmp_dir = str(Env.get().tmp_dir)
-        if file_url is None:
-            return None
-        parsed = urllib.parse.urlparse(file_url)
-        # We should never be passed a local file path here. The "len > 1" ensures that Windows
-        # file paths aren't mistaken for URLs with a single-character scheme.
-        assert len(parsed.scheme) > 1
-        if parsed.scheme != 'file':
-            # remote url
-            return file_url
-        file_path = urllib.parse.unquote(urllib.request.url2pathname(parsed.path))
-        if not file_path.startswith(pxt_tmp_dir):
-            # not a tmp file
-            return file_url
-        _, ext = os.path.splitext(file_path)
-        new_path = str(MediaStore.prepare_media_path(self.tbl_version.id, col.id, v_min, ext=ext))
-        os.rename(file_path, new_path)
-        new_file_url = urllib.parse.urljoin('file:', urllib.request.pathname2url(new_path))
-        return new_file_url
-
-    def _move_tmp_media_files(
-        self, table_rows: list[dict[str, Any]], media_cols: list[catalog.Column], v_min: int
-    ) -> None:
-        """Move tmp media files that we generated to a permanent location"""
-        for c in media_cols:
-            for table_row in table_rows:
-                file_url = table_row[c.store_name()]
-                table_row[c.store_name()] = self._move_tmp_media_file(file_url, c, v_min)
-
-    def _create_table_row(
-        self, input_row: exprs.DataRow, row_builder: exprs.RowBuilder, exc_col_ids: set[int], pk: tuple[int, ...]
-    ) -> tuple[dict[str, Any], int]:
-        """Return Tuple[complete table row, # of exceptions] for insert()
-        Creates a row that includes the PK columns, with the values from input_row.pk.
-        Returns:
-            Tuple[complete table row, # of exceptions]
-        """
-        table_row, num_excs = row_builder.create_table_row(input_row, exc_col_ids)
-        assert len(pk) == len(self._pk_cols)
-        for pk_col, pk_val in zip(self._pk_cols, pk):
-            table_row[pk_col.name] = pk_val
-        return table_row, num_excs
-
     def count(self) -> int:
         """Return the number of rows visible in self.tbl_version"""
         stmt = (
@@ -180,15 +137,123 @@ class StoreBase:
         assert isinstance(result, int)
         return result
 
+    def _exec_if_not_exists(self, stmt: str, wait_for_table: bool) -> None:
+        """
+        Execute a statement containing 'IF NOT EXISTS' and ignore any duplicate object-related errors.
+
+        The statement needs to run in a separate transaction, because the expected error conditions will abort the
+        enclosing transaction (and the ability to run additional statements in that same transaction).
+        """
+        while True:
+            with Env.get().begin_xact(for_write=True) as conn:
+                try:
+                    if wait_for_table and not Env.get().is_using_cockroachdb:
+                        # Try to lock the table to make sure that it exists. This needs to run in the same transaction
+                        # as 'stmt' to avoid a race condition.
+                        # TODO: adapt this for CockroachDB
+                        lock_stmt = f'LOCK TABLE {self._storage_name()} IN ACCESS EXCLUSIVE MODE'
+                        conn.execute(sql.text(lock_stmt))
+                    conn.execute(sql.text(stmt))
+                    return
+                except (sql.exc.IntegrityError, sql.exc.ProgrammingError) as e:
+                    Env.get().console_logger.info(f'{stmt} failed with: {e}')
+                    if (
+                        isinstance(e.orig, psycopg.errors.UniqueViolation)
+                        and 'duplicate key value violates unique constraint' in str(e.orig)
+                    ) or (
+                        isinstance(e.orig, (psycopg.errors.DuplicateObject, psycopg.errors.DuplicateTable))
+                        and 'already exists' in str(e.orig)
+                    ):
+                        # table already exists
+                        return
+                    elif isinstance(e.orig, psycopg.errors.UndefinedTable):
+                        # the Lock Table failed because the table doesn't exist yet; try again
+                        time.sleep(1)
+                        continue
+                    else:
+                        raise
+
+    def _store_tbl_exists(self) -> bool:
+        """Returns True if the store table exists, False otherwise."""
+        with Env.get().begin_xact(for_write=False) as conn:
+            q = (
+                'SELECT COUNT(*) FROM pg_catalog.pg_tables '
+                f"WHERE schemaname = 'public' AND tablename = {self._storage_name()!r}"
+            )
+            res = conn.execute(sql.text(q)).scalar_one()
+            return res == 1
+
     def create(self) -> None:
-        conn = Env.get().conn
-        self.sa_md.create_all(bind=conn)
+        """
+        Create or update store table to bring it in sync with self.sa_tbl. Idempotent.
+
+        This runs a sequence of DDL statements (Create Table, Alter Table Add Column, Create Index), each of which
+        is run in its own transaction.
+
+        The exception to that are local replicas, for which TableRestorer creates an enclosing transaction. In theory,
+        this should avoid the potential for race conditions that motivate the error handling present in
+        _exec_if_not_exists() (meaning: we shouldn't see those errors when creating local replicas).
+        TODO: remove the special case for local replicas in order to make the logic easier to reason about.
+        """
+        postgres_dialect = sql.dialects.postgresql.dialect()
+
+        if not self._store_tbl_exists():
+            # run Create Table If Not Exists; we always need If Not Exists to avoid race conditions between concurrent
+            # Pixeltable processes
+            create_stmt = sql.schema.CreateTable(self.sa_tbl, if_not_exists=True).compile(dialect=postgres_dialect)
+            self._exec_if_not_exists(str(create_stmt), wait_for_table=False)
+        else:
+            # ensure that all columns exist by running Alter Table Add Column If Not Exists for all columns
+            for col in self.sa_tbl.columns:
+                stmt = self._add_column_stmt(col)
+                self._exec_if_not_exists(stmt, wait_for_table=True)
+            # TODO: do we also need to ensure that these columns are now visible (ie, is there another potential race
+            # condition here?)
+
+        # ensure that all system indices exist by running Create Index If Not Exists
+        for idx in self.sa_tbl.indexes:
+            create_idx_stmt = sql.schema.CreateIndex(idx, if_not_exists=True).compile(dialect=postgres_dialect)
+            self._exec_if_not_exists(str(create_idx_stmt), wait_for_table=True)
+
+        # ensure that all visible non-system indices exist by running appropriate create statements
+        for id in self.tbl_version.get().idxs:
+            self.create_index(id)
+
+    def create_index(self, idx_id: int) -> None:
+        """Create If Not Exists for this index"""
+        idx_info = self.tbl_version.get().idxs[idx_id]
+        stmt = idx_info.idx.sa_create_stmt(self.tbl_version.get()._store_idx_name(idx_id), idx_info.val_col.sa_col)
+        self._exec_if_not_exists(str(stmt), wait_for_table=True)
+
+    def validate(self) -> None:
+        """Validate store table against self.table_version"""
+        with Env.get().begin_xact() as conn:
+            # check that all columns are present
+            q = f'SELECT column_name FROM information_schema.columns WHERE table_name = {self._storage_name()!r}'
+            store_col_info = {row[0] for row in conn.execute(sql.text(q)).fetchall()}
+            tbl_col_info = {col.store_name() for col in self.tbl_version.get().cols if col.is_stored}
+            assert tbl_col_info.issubset(store_col_info)
+
+            # check that all visible indices are present
+            q = f'SELECT indexname FROM pg_indexes WHERE tablename = {self._storage_name()!r}'
+            store_idx_names = {row[0] for row in conn.execute(sql.text(q)).fetchall()}
+            tbl_index_names = {
+                self.tbl_version.get()._store_idx_name(info.id) for info in self.tbl_version.get().idxs.values()
+            }
+            assert tbl_index_names.issubset(store_idx_names)
 
     def drop(self) -> None:
         """Drop store table"""
         conn = Env.get().conn
         self.sa_md.drop_all(bind=conn)
 
+    def _add_column_stmt(self, sa_col: sql.Column) -> str:
+        col_type_str = sa_col.type.compile(dialect=sql.dialects.postgresql.dialect())
+        return (
+            f'ALTER TABLE {self._storage_name()} ADD COLUMN IF NOT EXISTS '
+            f'{sa_col.name} {col_type_str} {"NOT " if not sa_col.nullable else ""} NULL'
+        )
+
     def add_column(self, col: catalog.Column) -> None:
         """Add column(s) to the store-resident table based on a catalog column
 
@@ -197,14 +262,13 @@ class StoreBase:
         """
         assert col.is_stored
         conn = Env.get().conn
-        col_type_str = col.get_sa_col_type().compile(dialect=conn.dialect)
+        col_type_str = col.sa_col_type.compile(dialect=conn.dialect)
         s_txt = f'ALTER TABLE {self._storage_name()} ADD COLUMN {col.store_name()} {col_type_str} NULL'
         added_storage_cols = [col.store_name()]
-        if col.records_errors:
-            # we also need to create the errormsg and errortype storage cols
-            s_txt += f' , ADD COLUMN {col.errormsg_store_name()} VARCHAR DEFAULT NULL'
-            s_txt += f' , ADD COLUMN {col.errortype_store_name()} VARCHAR DEFAULT NULL'
-            added_storage_cols.extend([col.errormsg_store_name(), col.errortype_store_name()])
+        if col.stores_cellmd:
+            cellmd_type_str = col.sa_cellmd_type().compile(dialect=conn.dialect)
+            s_txt += f' , ADD COLUMN {col.cellmd_store_name()} {cellmd_type_str} DEFAULT NULL'
+            added_storage_cols.append(col.cellmd_store_name())
 
         stmt = sql.text(s_txt)
         log_stmt(_logger, stmt)
@@ -215,25 +279,13 @@ class StoreBase:
     def drop_column(self, col: catalog.Column) -> None:
         """Execute Alter Table Drop Column statement"""
         s_txt = f'ALTER TABLE {self._storage_name()} DROP COLUMN {col.store_name()}'
-        if col.records_errors:
-            s_txt += f' , DROP COLUMN {col.errormsg_store_name()}'
-            s_txt += f' , DROP COLUMN {col.errortype_store_name()}'
+        if col.stores_cellmd:
+            s_txt += f' , DROP COLUMN {col.cellmd_store_name()}'
         stmt = sql.text(s_txt)
         log_stmt(_logger, stmt)
         Env.get().conn.execute(stmt)
 
-    def ensure_columns_exist(self, cols: Iterable[catalog.Column]) -> None:
-        conn = Env.get().conn
-        sql_text = f'SELECT column_name FROM information_schema.columns WHERE table_name = {self._storage_name()!r}'
-        result = conn.execute(sql.text(sql_text))
-        existing_cols = {row[0] for row in result}
-        for col in cols:
-            if col.store_name() not in existing_cols:
-                self.add_column(col)
-
-    def load_column(
-        self, col: catalog.Column, exec_plan: ExecNode, value_expr_slot_idx: int, on_error: Literal['abort', 'ignore']
-    ) -> int:
+    def load_column(self, col: catalog.Column, exec_plan: ExecNode, abort_on_exc: bool) -> int:
         """Update store column of a computed column with values produced by an execution plan
 
         Returns:
@@ -242,84 +294,69 @@ class StoreBase:
             sql.exc.DBAPIError if there was a SQL error during execution
             excs.Error if on_error='abort' and there was an exception during row evaluation
         """
-        assert col.tbl.id == self.tbl_version.id
+        assert col.get_tbl().id == self.tbl_version.id
         num_excs = 0
         num_rows = 0
         # create temp table to store output of exec_plan, with the same primary key as the store table
         tmp_name = f'temp_{self._storage_name()}'
-        tmp_pk_cols = [sql.Column(col.name, col.type, primary_key=True) for col in self.pk_columns()]
-        tmp_cols = tmp_pk_cols.copy()
+        tmp_pk_cols = tuple(sql.Column(col.name, col.type, primary_key=True) for col in self.pk_columns())
         tmp_val_col = sql.Column(col.sa_col.name, col.sa_col.type)
-        tmp_cols.append(tmp_val_col)
+        tmp_cols = [*tmp_pk_cols, tmp_val_col]
         # add error columns if the store column records errors
-        if col.records_errors:
-            tmp_errortype_col = sql.Column(col.sa_errortype_col.name, col.sa_errortype_col.type)
-            tmp_cols.append(tmp_errortype_col)
-            tmp_errormsg_col = sql.Column(col.sa_errormsg_col.name, col.sa_errormsg_col.type)
-            tmp_cols.append(tmp_errormsg_col)
+        if col.stores_cellmd:
+            tmp_cellmd_col = sql.Column(col.sa_cellmd_col.name, col.sa_cellmd_col.type)
+            tmp_cols.append(tmp_cellmd_col)
+        tmp_col_names = [col.name for col in tmp_cols]
+
         tmp_tbl = sql.Table(tmp_name, self.sa_md, *tmp_cols, prefixes=['TEMPORARY'])
         conn = Env.get().conn
         tmp_tbl.create(bind=conn)
 
+        row_builder = exec_plan.row_builder
+
         try:
+            table_rows: list[tuple[Any]] = []
+
             # insert rows from exec_plan into temp table
-            # TODO: unify the table row construction logic with RowBuilder.create_table_row()
             for row_batch in exec_plan:
                 num_rows += len(row_batch)
-                tbl_rows: list[dict[str, Any]] = []
-                for result_row in row_batch:
-                    tbl_row: dict[str, Any] = {}
-                    for pk_col, pk_val in zip(self.pk_columns(), result_row.pk):
-                        tbl_row[pk_col.name] = pk_val
-
-                    if col.is_computed:
-                        if result_row.has_exc(value_expr_slot_idx):
-                            num_excs += 1
-                            value_exc = result_row.get_exc(value_expr_slot_idx)
-                            if on_error == 'abort':
-                                raise excs.Error(
-                                    f'Error while evaluating computed column `{col.name}`:\n{value_exc}'
-                                ) from value_exc
-                            # we store a NULL value and record the exception/exc type
-                            error_type = type(value_exc).__name__
-                            error_msg = str(value_exc)
-                            tbl_row[col.sa_col.name] = None
-                            tbl_row[col.sa_errortype_col.name] = error_type
-                            tbl_row[col.sa_errormsg_col.name] = error_msg
-                        else:
-                            if col.col_type.is_image_type() and result_row.file_urls[value_expr_slot_idx] is None:
-                                # we have yet to store this image
-                                filepath = str(MediaStore.prepare_media_path(col.tbl.id, col.id, col.tbl.version))
-                                result_row.flush_img(value_expr_slot_idx, filepath)
-                            val = result_row.get_stored_val(value_expr_slot_idx, col.sa_col.type)
-                            if col.col_type.is_media_type():
-                                val = self._move_tmp_media_file(val, col, result_row.pk[-1])
-                            tbl_row[col.sa_col.name] = val
-                            if col.records_errors:
-                                tbl_row[col.sa_errortype_col.name] = None
-                                tbl_row[col.sa_errormsg_col.name] = None
-
-                    tbl_rows.append(tbl_row)
-                conn.execute(sql.insert(tmp_tbl), tbl_rows)
+                batch_table_rows: list[tuple[Any]] = []
+
+                for row in row_batch:
+                    if abort_on_exc and row.has_exc():
+                        exc = row.get_first_exc()
+                        raise excs.Error(f'Error while evaluating computed column {col.name!r}:\n{exc}') from exc
+                    table_row, num_row_exc = row_builder.create_store_table_row(row, None, row.pk)
+                    num_excs += num_row_exc
+                    batch_table_rows.append(tuple(table_row))
+
+                table_rows.extend(batch_table_rows)
+
+                if len(table_rows) >= self.__INSERT_BATCH_SIZE:
+                    self.sql_insert(tmp_tbl, tmp_col_names, table_rows)
+                    table_rows.clear()
+
+            if len(table_rows) > 0:
+                self.sql_insert(tmp_tbl, tmp_col_names, table_rows)
 
             # update store table with values from temp table
             update_stmt = sql.update(self.sa_tbl)
             for pk_col, tmp_pk_col in zip(self.pk_columns(), tmp_pk_cols):
                 update_stmt = update_stmt.where(pk_col == tmp_pk_col)
             update_stmt = update_stmt.values({col.sa_col: tmp_val_col})
-            if col.records_errors:
-                update_stmt = update_stmt.values(
-                    {col.sa_errortype_col: tmp_errortype_col, col.sa_errormsg_col: tmp_errormsg_col}
-                )
+            if col.stores_cellmd:
+                update_stmt = update_stmt.values({col.sa_cellmd_col: tmp_cellmd_col})
             log_explain(_logger, update_stmt, conn)
             conn.execute(update_stmt)
+
         finally:
 
             def remove_tmp_tbl() -> None:
                 self.sa_md.remove(tmp_tbl)
                 tmp_tbl.drop(bind=conn)
 
-            run_cleanup(remove_tmp_tbl, raise_error=True)
+            run_cleanup(remove_tmp_tbl, raise_error=False)
+
         return num_excs
 
     def insert_rows(
@@ -327,9 +364,9 @@ class StoreBase:
         exec_plan: ExecNode,
         v_min: int,
         show_progress: bool = True,
-        rowids: Optional[Iterator[int]] = None,
+        rowids: Iterator[int] | None = None,
         abort_on_exc: bool = False,
-    ) -> tuple[int, int, set[int]]:
+    ) -> tuple[set[int], RowCountStats]:
         """Insert rows into the store table and update the catalog table's md
         Returns:
             number of inserted rows, number of exceptions, set of column ids that have exceptions
@@ -339,53 +376,80 @@ class StoreBase:
         num_excs = 0
         num_rows = 0
         cols_with_excs: set[int] = set()
-        progress_bar: Optional[tqdm] = None  # create this only after we started executing
+        progress_bar: tqdm | None = None  # create this only after we started executing
         row_builder = exec_plan.row_builder
-        media_cols = [info.col for info in row_builder.table_columns if info.col.col_type.is_media_type()]
-        conn = Env.get().conn
+
+        store_col_names = row_builder.store_column_names()
 
         try:
+            table_rows: list[tuple[Any]] = []
             exec_plan.open()
+
             for row_batch in exec_plan:
                 num_rows += len(row_batch)
-                for batch_start_idx in range(0, len(row_batch), self.__INSERT_BATCH_SIZE):
-                    # compute batch of rows and convert them into table rows
-                    table_rows: list[dict[str, Any]] = []
-                    batch_stop_idx = min(batch_start_idx + self.__INSERT_BATCH_SIZE, len(row_batch))
-                    for row_idx in range(batch_start_idx, batch_stop_idx):
-                        row = row_batch[row_idx]
-                        # if abort_on_exc == True, we need to check for media validation exceptions
-                        if abort_on_exc and row.has_exc():
-                            exc = row.get_first_exc()
-                            raise exc
-
-                        rowid = (next(rowids),) if rowids is not None else row.pk[:-1]
-                        pk = (*rowid, v_min)
-                        table_row, num_row_exc = self._create_table_row(row, row_builder, cols_with_excs, pk=pk)
-                        num_excs += num_row_exc
-                        table_rows.append(table_row)
-
-                        if show_progress:
-                            if progress_bar is None:
-                                warnings.simplefilter('ignore', category=TqdmWarning)
-                                progress_bar = tqdm(
-                                    desc=f'Inserting rows into `{self.tbl_version.get().name}`',
-                                    unit=' rows',
-                                    ncols=100,
-                                    file=sys.stdout,
-                                )
-                            progress_bar.update(1)
-
-                    # insert batch of rows
-                    self._move_tmp_media_files(table_rows, media_cols, v_min)
-                    conn.execute(sql.insert(self.sa_tbl), table_rows)
+                batch_table_rows: list[tuple[Any]] = []
+
+                # compute batch of rows and convert them into table rows
+                for row in row_batch:
+                    # if abort_on_exc == True, we need to check for media validation exceptions
+                    if abort_on_exc and row.has_exc():
+                        exc = row.get_first_exc()
+                        raise exc
+
+                    rowid = (next(rowids),) if rowids is not None else row.pk[:-1]
+                    pk = (*rowid, v_min)
+                    assert len(pk) == len(self._pk_cols)
+                    table_row, num_row_exc = row_builder.create_store_table_row(row, cols_with_excs, pk)
+                    num_excs += num_row_exc
+
+                    if show_progress and Env.get().verbosity >= 1:
+                        if progress_bar is None:
+                            warnings.simplefilter('ignore', category=TqdmWarning)
+                            progress_bar = tqdm(
+                                desc=f'Inserting rows into `{self.tbl_version.get().name}`',
+                                unit=' rows',
+                                ncols=100,
+                                file=sys.stdout,
+                            )
+                        progress_bar.update(1)
+
+                    batch_table_rows.append(tuple(table_row))
+
+                table_rows.extend(batch_table_rows)
+
+                # if a batch is ready for insertion into the database, insert it
+                if len(table_rows) >= self.__INSERT_BATCH_SIZE:
+                    self.sql_insert(self.sa_tbl, store_col_names, table_rows)
+                    table_rows.clear()
+
+            # insert any remaining rows
+            if len(table_rows) > 0:
+                self.sql_insert(self.sa_tbl, store_col_names, table_rows)
+
             if progress_bar is not None:
                 progress_bar.close()
-            return num_rows, num_excs, cols_with_excs
+            computed_values = exec_plan.ctx.num_computed_exprs * num_rows
+            row_counts = RowCountStats(ins_rows=num_rows, num_excs=num_excs, computed_values=computed_values)
+
+            return cols_with_excs, row_counts
         finally:
             exec_plan.close()
 
-    def _versions_clause(self, versions: list[Optional[int]], match_on_vmin: bool) -> sql.ColumnElement[bool]:
+    @classmethod
+    def sql_insert(cls, sa_tbl: sql.Table, store_col_names: list[str], table_rows: list[tuple[Any]]) -> None:
+        assert len(table_rows) > 0
+        conn = Env.get().conn
+        conn.execute(sql.insert(sa_tbl), [dict(zip(store_col_names, table_row)) for table_row in table_rows])
+
+        # TODO: Inserting directly via psycopg delivers a small performance benefit, but is somewhat fraught due to
+        #     differences in the data representation that SQLAlchemy/psycopg expect. The below code will do the
+        #     insertion in psycopg and can be used if/when we decide to pursue that optimization.
+        # col_names_str = ", ".join(store_col_names)
+        # placeholders_str = ", ".join('%s' for _ in store_col_names)
+        # stmt_text = f'INSERT INTO {self.sa_tbl.name} ({col_names_str}) VALUES ({placeholders_str})'
+        # conn.exec_driver_sql(stmt_text, table_rows)
+
+    def _versions_clause(self, versions: list[int | None], match_on_vmin: bool) -> sql.ColumnElement[bool]:
         """Return filter for base versions"""
         v = versions[0]
         if v is None:
@@ -403,9 +467,9 @@ class StoreBase:
     def delete_rows(
         self,
         current_version: int,
-        base_versions: list[Optional[int]],
+        base_versions: list[int | None],
         match_on_vmin: bool,
-        where_clause: Optional[sql.ColumnElement[bool]],
+        where_clause: sql.ColumnElement[bool] | None,
     ) -> int:
         """Mark rows as deleted that are live and were created prior to current_version.
         Also: populate the undo columns
@@ -424,7 +488,7 @@ class StoreBase:
         base_versions_clause = (
             sql.true() if len(base_versions) == 0 else self.base._versions_clause(base_versions, match_on_vmin)
         )
-        set_clause: dict[sql.Column, Union[int, sql.Column]] = {self.v_max_col: current_version}
+        set_clause: dict[sql.Column, int | sql.Column] = {self.v_max_col: current_version}
         for index_info in self.tbl_version.get().idxs_by_name.values():
             # copy value column to undo column
             set_clause[index_info.undo_col.sa_col] = index_info.val_col.sa_col
@@ -451,8 +515,7 @@ class StoreBase:
             *[c1 == c2 for c1, c2 in zip(self.rowid_columns(), filter_view.rowid_columns())],
         )
         stmt = (
-            sql.select('*')  # TODO: Use a more specific list of columns?
-            .select_from(self.sa_tbl)
+            sql.select(self.sa_tbl)
             .where(self.v_min_col <= version)
             .where(self.v_max_col > version)
             .where(sql.exists().where(filter_predicate))
@@ -532,7 +595,7 @@ class StoreComponentView(StoreView):
         self.rowid_cols.append(self.pos_col)
         return self.rowid_cols
 
-    def create_sa_tbl(self, tbl_version: Optional[catalog.TableVersion] = None) -> None:
+    def create_sa_tbl(self, tbl_version: catalog.TableVersion | None = None) -> None:
         if tbl_version is None:
             tbl_version = self.tbl_version.get()
         super().create_sa_tbl(tbl_version)