pixeltable 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

pixeltable/catalog/view.py

@@ -17,11 +17,12 @@ if TYPE_CHECKING:
 
 
 from .column import Column
-from .globals import _POS_COLUMN_NAME, MediaValidation, UpdateStatus
+from .globals import _POS_COLUMN_NAME, MediaValidation
 from .table import Table
 from .table_version import TableVersion
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
     from pixeltable.globals import TableDataSource
@@ -229,7 +230,10 @@ class View(Table):
 
             try:
                 plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
-                _, status = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+                _, row_counts = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+                status = UpdateStatus(row_count_stats=row_counts)
+                tbl_version._write_md_update_status(0, update_status=status)
+
             except:
                 # we need to remove the orphaned TableVersion instance
                 del catalog.Catalog.get()._tbl_versions[tbl_version.id, tbl_version.effective_version]
@@ -275,6 +279,9 @@ class View(Table):
         md = super()._get_metadata()
         md['is_view'] = True
         md['is_snapshot'] = self._tbl_version_path.is_snapshot()
+        base_tbl = self._get_base_table()
+        base_version = self._effective_base_versions[0]
+        md['base'] = base_tbl._path() if base_version is None else f'{base_tbl._path()}:{base_version}'
         return md
 
     def insert(

pixeltable/config.py

@@ -25,19 +25,26 @@ class Config:
 
     __home: Path
     __config_file: Path
+    __config_overrides: dict[str, Any]
     __config_dict: dict[str, Any]
 
-    def __init__(self) -> None:
+    def __init__(self, config_overrides: dict[str, Any]) -> None:
         assert self.__instance is None, 'Config is a singleton; use Config.get() to access the instance'
 
-        self.__home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
+        for var in config_overrides:
+            if var not in KNOWN_CONFIG_OVERRIDES:
+                raise excs.Error(f'Unrecognized configuration variable: {var}')
+
+        self.__config_overrides = config_overrides
+
+        self.__home = Path(self.lookup_env('pixeltable', 'home', str(Path.home() / '.pixeltable')))
         if self.__home.exists() and not self.__home.is_dir():
-            raise RuntimeError(f'{self.__home} is not a directory')
+            raise excs.Error(f'Not a directory: {self.__home}')
         if not self.__home.exists():
             print(f'Creating a Pixeltable instance at: {self.__home}')
             self.__home.mkdir()
 
-        self.__config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self.__home / 'config.toml')))
+        self.__config_file = Path(self.lookup_env('pixeltable', 'config', str(self.__home / 'config.toml')))
 
         self.__config_dict: dict[str, Any]
         if os.path.isfile(self.__config_file):
@@ -46,6 +53,12 @@ class Config:
                     self.__config_dict = toml.load(stream)
                 except Exception as exc:
                     raise excs.Error(f'Could not read config file: {self.__config_file}') from exc
+            for section, section_dict in self.__config_dict.items():
+                if section not in KNOWN_CONFIG_OPTIONS:
+                    raise excs.Error(f'Unrecognized section {section!r} in config file: {self.__config_file}')
+                for key in section_dict:
+                    if key not in KNOWN_CONFIG_OPTIONS[section]:
+                        raise excs.Error(f"Unrecognized option '{section}.{key}' in config file: {self.__config_file}")
         else:
             self.__config_dict = self.__create_default_config(self.__config_file)
             with open(self.__config_file, 'w', encoding='utf-8') as stream:
@@ -65,10 +78,18 @@ class Config:
 
     @classmethod
     def get(cls) -> Config:
-        if cls.__instance is None:
-            cls.__instance = cls()
+        cls.init({})
         return cls.__instance
 
+    @classmethod
+    def init(cls, config_overrides: dict[str, Any]) -> None:
+        if cls.__instance is None:
+            cls.__instance = cls(config_overrides)
+        elif len(config_overrides) > 0:
+            raise excs.Error(
+                'Pixeltable has already been initialized; cannot specify new config values in the same session'
+            )
+
     @classmethod
     def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
         free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
@@ -76,14 +97,23 @@ class Config:
         file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
         return {'pixeltable': {'file_cache_size_g': round(file_cache_size_g, 1), 'hide_warnings': False}}
 
-    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+    def lookup_env(self, section: str, key: str, default: Any = None) -> Any:
+        override_var = f'{section}.{key}'
         env_var = f'{section.upper()}_{key.upper()}'
+        if override_var in self.__config_overrides:
+            return self.__config_overrides[override_var]
         if env_var in os.environ:
-            value = os.environ[env_var]
-        elif section in self.__config_dict and key in self.__config_dict[section]:
+            return os.environ[env_var]
+        return default
+
+    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+        value = self.lookup_env(section, key)  # Try to get from environment first
+        # Next try the config file
+        if value is None and section in self.__config_dict and key in self.__config_dict[section]:
             value = self.__config_dict[section][key]
-        else:
-            return None
+
+        if value is None:
+            return None  # Not specified
 
         try:
             if expected_type is bool and isinstance(value, str):
@@ -91,7 +121,7 @@ class Config:
                     raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
                 return value.lower() == 'true'  # type: ignore[return-value]
             return expected_type(value)  # type: ignore[call-arg]
-        except ValueError as exc:
+        except (ValueError, TypeError) as exc:
             raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}') from exc
 
     def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
@@ -105,3 +135,37 @@ class Config:
 
     def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
         return self.get_value(key, bool, section)
+
+
+KNOWN_CONFIG_OPTIONS = {
+    'pixeltable': {
+        'home': 'Path to the Pixeltable home directory',
+        'config': 'Path to the Pixeltable config file',
+        'pgdata': 'Path to the Pixeltable postgres data directory',
+        'db': 'Postgres database name',
+        'file_cache_size_g': 'Size of the file cache in GB',
+        'time_zone': 'Default time zone for timestamps',
+        'hide_warnings': 'Hide warnings from the console',
+        'verbosity': 'Verbosity level for console output',
+        'api_key': 'API key for Pixeltable cloud',
+    },
+    'anthropic': {'api_key': 'Anthropic API key'},
+    'bedrock': {'api_key': 'AWS Bedrock API key'},
+    'deepseek': {'api_key': 'Deepseek API key'},
+    'fireworks': {'api_key': 'Fireworks API key'},
+    'gemini': {'api_key': 'Gemini API key'},
+    'groq': {'api_key': 'Groq API key'},
+    'label_studio': {'api_key': 'Label Studio API key', 'url': 'Label Studio server URL'},
+    'mistral': {'api_key': 'Mistral API key'},
+    'openai': {'api_key': 'OpenAI API key'},
+    'replicate': {'api_token': 'Replicate API token'},
+    'together': {'api_key': 'Together API key'},
+    'pypi': {'api_key': 'PyPI API key (for internal use only)'},
+}
+
+
+KNOWN_CONFIG_OVERRIDES = {
+    f'{section}.{key}': info
+    for section, section_dict in KNOWN_CONFIG_OPTIONS.items()
+    for key, info in section_dict.items()
+}

pixeltable/dataframe.py

@@ -15,7 +15,7 @@ import sqlalchemy as sql
 
 from pixeltable import catalog, exceptions as excs, exec, exprs, plan, type_system as ts
 from pixeltable.catalog import Catalog, is_valid_identifier
-from pixeltable.catalog.globals import UpdateStatus
+from pixeltable.catalog.update_status import UpdateStatus
 from pixeltable.env import Env
 from pixeltable.plan import Planner, SampleClause
 from pixeltable.type_system import ColumnType

pixeltable/env.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import datetime
 import glob
 import http.server
@@ -22,6 +23,7 @@ from sys import stdout
 from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
+import nest_asyncio  # type: ignore[import-untyped]
 import pixeltable_pgserver
 import sqlalchemy as sql
 from pillow_heif import register_heif_opener  # type: ignore[import-untyped]
@@ -85,6 +87,7 @@ class Env:
     _current_conn: Optional[sql.Connection]
     _current_session: Optional[sql.orm.Session]
     _dbms: Optional[Dbms]
+    _event_loop: Optional[asyncio.AbstractEventLoop]  # event loop for ExecNode
 
     @classmethod
     def get(cls) -> Env:
@@ -140,6 +143,32 @@ class Env:
         self._current_conn = None
         self._current_session = None
         self._dbms = None
+        self._event_loop = None
+
+    def _init_event_loop(self) -> None:
+        try:
+            # check if we are already in an event loop (eg, Jupyter's); if so, patch it to allow
+            # multiple run_until_complete()
+            running_loop = asyncio.get_running_loop()
+            self._event_loop = running_loop
+            _logger.debug('Patched running loop')
+        except RuntimeError:
+            self._event_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._event_loop)
+            # we set a deliberately long duration to avoid warnings getting printed to the console in debug mode
+            self._event_loop.slow_callback_duration = 3600
+
+        # always allow nested event loops, we need that to run async udfs synchronously (eg, for SimilarityExpr);
+        # see run_coroutine_synchronously()
+        nest_asyncio.apply()
+        if _logger.isEnabledFor(logging.DEBUG):
+            self._event_loop.set_debug(True)
+
+    @property
+    def event_loop(self) -> asyncio.AbstractEventLoop:
+        if self._event_loop is None:
+            self._init_event_loop()
+        return self._event_loop
 
     @property
     def db_url(self) -> str:

pixeltable/exec/exec_node.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
 import abc
-import asyncio
 import logging
 from typing import AsyncIterator, Iterable, Iterator, Optional, TypeVar
 
 from pixeltable import exprs
+from pixeltable.env import Env
 
 from .data_row_batch import DataRowBatch
 from .exec_context import ExecContext
@@ -59,26 +59,7 @@ class ExecNode(abc.ABC):
         pass
 
     def __iter__(self) -> Iterator[DataRowBatch]:
-        running_loop: Optional[asyncio.AbstractEventLoop] = None
-        loop: asyncio.AbstractEventLoop
-        try:
-            # check if we are already in an event loop (eg, Jupyter's); if so, patch it to allow
-            # multiple run_until_complete()
-            running_loop = asyncio.get_running_loop()
-            import nest_asyncio  # type: ignore[import-untyped]
-
-            nest_asyncio.apply()
-            loop = running_loop
-            _logger.debug('Patched running loop')
-        except RuntimeError:
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            # we set a deliberately long duration to avoid warnings getting printed to the console in debug mode
-            loop.slow_callback_duration = 3600
-
-        if _logger.isEnabledFor(logging.DEBUG):
-            loop.set_debug(True)
-
+        loop = Env.get().event_loop
         aiter = self.__aiter__()
         try:
             while True:
@@ -86,9 +67,11 @@ class ExecNode(abc.ABC):
                 yield batch
         except StopAsyncIteration:
             pass
-        finally:
-            if loop != running_loop:
-                loop.close()
+        # TODO:
+        #  - we seem to have some tasks that aren't accounted for by ExprEvalNode and don't get cancelled by the time
+        #    we end up here
+        # - however, blindly cancelling all pending tasks doesn't work when running in a jupyter environment, which
+        #   creates tasks on its own
 
     def open(self) -> None:
         """Bottom-up initialization of nodes for execution. Must be called before __next__."""

pixeltable/exec/expr_eval/schedulers.py

@@ -4,9 +4,10 @@ import asyncio
 import datetime
 import inspect
 import logging
+import re
 import sys
 import time
-from typing import Awaitable, Collection, Optional
+from typing import Any, Awaitable, Collection, Optional
 
 from pixeltable import env, func
 from pixeltable.config import Config
@@ -250,8 +251,20 @@ class RequestRateScheduler(Scheduler):
     total_retried: int
 
     TIME_FORMAT = '%H:%M.%S %f'
-    MAX_RETRIES = 10
+    MAX_RETRIES = 3
     DEFAULT_RATE_LIMIT = 600  # requests per minute
+    RATE_LIMIT_INDICATORS = ('rate limit', 'too many requests', '429', 'quota exceeded', 'throttled', 'rate exceeded')
+    RETRY_AFTER_PATTERNS = (
+        r'retry after (\d+(?:\.\d+)?)\s*seconds?',
+        r'try again in (\d+(?:\.\d+)?)\s*seconds?',
+        r'wait (\d+(?:\.\d+)?)\s*seconds?',
+        r'retry-after:\s*(\d+(?:\.\d+)?)',
+    )
+
+    # Exponential backoff defaults
+    BASE_RETRY_DELAY = 1.0  # in seconds
+    MAX_RETRY_DELAY = 60.0  # in seconds
+    RETRY_BACKOFF_MULTIPLIER = 2.0
 
     def __init__(self, resource_pool: str, dispatcher: Dispatcher):
         super().__init__(resource_pool, dispatcher)
@@ -337,11 +350,12 @@ class RequestRateScheduler(Scheduler):
             self.dispatcher.dispatch(request.rows, exec_ctx)
 
         except Exception as exc:
-            # TODO: which exception can be retried?
-            _logger.debug(f'exception for {self.resource_pool}: {exc}')
-            status = getattr(exc, 'status', None)
-            _logger.debug(f'type={type(exc)} has_status={hasattr(exc, "status")} status={status}')
-            if num_retries < self.MAX_RETRIES:
+            _logger.debug(f'exception for {self.resource_pool}: type={type(exc)}\n{exc}')
+            is_rate_limit_error, retry_after = self._is_rate_limit_error(exc)
+            if is_rate_limit_error and num_retries < self.MAX_RETRIES:
+                retry_delay = self._compute_retry_delay(num_retries, retry_after)
+                _logger.debug(f'scheduler {self.resource_pool}: retrying after {retry_delay}')
+                await asyncio.sleep(retry_delay)
                 self.queue.put_nowait(self.QueueItem(request, num_retries + 1, exec_ctx))
                 return
 
@@ -358,6 +372,119 @@ class RequestRateScheduler(Scheduler):
             if is_task:
                 self.num_in_flight -= 1
 
+    def _is_rate_limit_error(self, exc: Exception) -> tuple[bool, Optional[float]]:
+        """Returns True if the exception indicates a rate limit error, and the retry delay in seconds."""
+        from http import HTTPStatus
+
+        # Check for HTTP status TOO_MANY_REQUESTS in various exception classes.
+        # We look for attributes that contain status codes, instead of checking the type of the exception,
+        # in order to handle a wider variety of exception classes.
+        is_rate_limit_error = False
+        retry_delay: Optional[float] = None
+
+        # requests.HTTPError/httpx.HTTPStatusError
+        if (
+            hasattr(exc, 'response')
+            and hasattr(exc.response, 'status_code')
+            and exc.response.status_code == HTTPStatus.TOO_MANY_REQUESTS.value
+        ):
+            is_rate_limit_error = True
+            retry_delay = self._extract_retry_delay_from_headers(exc.response.headers)
+        elif (
+            # urllib.error.HTTPError
+            (hasattr(exc, 'code') and exc.code == HTTPStatus.TOO_MANY_REQUESTS.value)
+            # aiohttp.ClientResponseError
+            or (hasattr(exc, 'status') and exc.status == HTTPStatus.TOO_MANY_REQUESTS.value)
+        ) and hasattr(exc, 'headers'):
+            is_rate_limit_error = True
+            retry_delay = self._extract_retry_delay_from_headers(exc.headers)
+
+        if is_rate_limit_error:
+            return True, retry_delay
+
+        # Check common rate limit keywords in exception message
+        error_msg = str(exc).lower()
+        if any(indicator in error_msg for indicator in self.RATE_LIMIT_INDICATORS):
+            retry_delay = self._extract_retry_delay_from_message(error_msg)
+            return True, retry_delay
+
+        return False, None
+
+    def _extract_retry_delay_from_headers(self, headers: Optional[Any]) -> Optional[float]:
+        """Extract retry delay from HTTP headers."""
+        if headers is None:
+            return None
+
+        # convert headers to dict-like object for consistent access
+        header_dict: dict
+        if hasattr(headers, 'get'):
+            header_dict = headers
+        else:
+            # headers are a list of tuples or other format
+            try:
+                header_dict = dict(headers)
+            except (TypeError, ValueError):
+                return None
+        # normalize dict keys: lowercase and remove dashes
+        header_dict = {k.lower().replace('-', ''): v for k, v in header_dict.items()}
+
+        # check Retry-After header
+        retry_after = header_dict.get('retryafter')
+        if retry_after is not None:
+            try:
+                return float(retry_after)
+            except (ValueError, TypeError):
+                pass
+
+        # check X-RateLimit-Reset (Unix timestamp)
+        reset_time = header_dict.get('xratelimitreset')
+        if reset_time is not None:
+            try:
+                reset_timestamp = float(reset_time)
+                delay = max(0, reset_timestamp - time.time())
+                return delay
+            except (ValueError, TypeError):
+                pass
+
+        # check X-RateLimit-Reset-After (seconds from now)
+        reset_after = header_dict.get('xratelimitresetafter')
+        if reset_after is not None:
+            try:
+                return float(reset_after)
+            except (ValueError, TypeError):
+                pass
+
+        return None
+
+    def _extract_retry_delay_from_message(self, msg: str) -> Optional[float]:
+        msg_lower = msg.lower()
+        for pattern in self.RETRY_AFTER_PATTERNS:
+            match = re.search(pattern, msg_lower)
+            if match is not None:
+                try:
+                    return float(match.group(1))
+                except (ValueError, TypeError):
+                    continue
+        return None
+
+    def _compute_retry_delay(self, num_retries: int, retry_after: Optional[float] = None) -> float:
+        """
+        Calculate exponential backoff delay for rate limit errors.
+
+        Args:
+            retry_count: Number of retries attempted (0-based)
+            retry_after: Suggested delay from Retry-After header
+
+        Returns:
+            Delay in seconds
+        """
+        if retry_after is not None and retry_after > 0:
+            # Use server-suggested delay, but cap it at max_delay
+            return max(min(retry_after, self.MAX_RETRY_DELAY), self.BASE_RETRY_DELAY)
+        else:
+            delay = self.BASE_RETRY_DELAY * (self.RETRY_BACKOFF_MULTIPLIER**num_retries)
+            return max(min(delay, self.MAX_RETRY_DELAY), self.BASE_RETRY_DELAY)
+
 
 # all concrete Scheduler subclasses that implement matches()
 SCHEDULERS = [RateLimitsScheduler, RequestRateScheduler]

pixeltable/exprs/column_property_ref.py

@@ -26,6 +26,7 @@ class ColumnPropertyRef(Expr):
         ERRORMSG = 1
         FILEURL = 2
         LOCALPATH = 3
+        CELLMD = 4  # JSON metadata for the cell, e.g. errortype, errormsg for media columns
 
     def __init__(self, col_ref: ColumnRef, prop: Property):
         super().__init__(ts.StringType(nullable=True))
@@ -51,8 +52,8 @@ class ColumnPropertyRef(Expr):
     def __repr__(self) -> str:
         return f'{self._col_ref}.{self.prop.name.lower()}'
 
-    def is_error_prop(self) -> bool:
-        return self.prop in (self.Property.ERRORTYPE, self.Property.ERRORMSG)
+    def is_cellmd_prop(self) -> bool:
+        return self.prop in (self.Property.ERRORTYPE, self.Property.ERRORMSG, self.Property.CELLMD)
 
     def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
         if not self._col_ref.col_handle.get().is_stored:
@@ -63,21 +64,27 @@ class ColumnPropertyRef(Expr):
         if (
             col.col_type.is_media_type()
             and col.media_validation == catalog.MediaValidation.ON_READ
-            and self.is_error_prop()
+            and self.is_cellmd_prop()
         ):
             return None
 
         if self.prop == self.Property.ERRORTYPE:
-            assert col.sa_errortype_col is not None
-            return col.sa_errortype_col
+            return col.sa_cellmd_col.op('->>')('errortype')
         if self.prop == self.Property.ERRORMSG:
-            assert col.sa_errormsg_col is not None
-            return col.sa_errormsg_col
+            return col.sa_cellmd_col.op('->>')('errormsg')
+        if self.prop == self.Property.CELLMD:
+            assert col.sa_cellmd_col is not None
+            return col.sa_cellmd_col
         if self.prop == self.Property.FILEURL:
             # the file url is stored as the column value
             return sql_elements.get(self._col_ref)
         return None
 
+    @classmethod
+    def create_cellmd_exc(cls, exc: Exception) -> dict[str, str]:
+        """Create a cellmd value from an exception."""
+        return {'errortype': type(exc).__name__, 'errormsg': str(exc)}
+
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
         if self.prop == self.Property.FILEURL:
             assert data_row.has_val[self._col_ref.slot_idx]
@@ -87,14 +94,19 @@ class ColumnPropertyRef(Expr):
             assert data_row.has_val[self._col_ref.slot_idx]
             data_row[self.slot_idx] = data_row.file_paths[self._col_ref.slot_idx]
             return
-        elif self.is_error_prop():
+        elif self.is_cellmd_prop():
             exc = data_row.get_exc(self._col_ref.slot_idx)
             if exc is None:
                 data_row[self.slot_idx] = None
             elif self.prop == self.Property.ERRORTYPE:
                 data_row[self.slot_idx] = type(exc).__name__
-            else:
+            elif self.prop == self.Property.ERRORMSG:
                 data_row[self.slot_idx] = str(exc)
+            elif self.prop == self.Property.CELLMD:
+                data_row[self.slot_idx] = self.create_cellmd_exc(exc)
+            else:
+                raise AssertionError(f'Unknown property {self.prop}')
+            return
         else:
             raise AssertionError()
 

pixeltable/exprs/column_ref.py

@@ -115,11 +115,15 @@ class ColumnRef(Expr):
         from .column_property_ref import ColumnPropertyRef
 
         # resolve column properties
+        if name == ColumnPropertyRef.Property.CELLMD.name.lower():
+            # This is not user accessible, but used internally to store cell metadata
+            return super().__getattr__(name)
+
         if (
             name == ColumnPropertyRef.Property.ERRORTYPE.name.lower()
             or name == ColumnPropertyRef.Property.ERRORMSG.name.lower()
         ):
-            property_is_present = self.col.is_stored and (self.col.is_computed or self.col_type.is_media_type())
+            property_is_present = self.col.stores_cellmd
             if not property_is_present:
                 raise excs.Error(f'{name} only valid for a stored computed or media column: {self}')
             return ColumnPropertyRef(self, ColumnPropertyRef.Property[name.upper()])

pixeltable/exprs/function_call.py

@@ -446,11 +446,11 @@ class FunctionCall(Expr):
                 dedent(
                     f"""
                     The UDF '{fn.self_path}' cannot be located, because
-                    {{errormsg}}
+                    {{error_msg}}
                     """
                 )
                 .strip()
-                .format(errormsg=fn.errormsg)
+                .format(error_msg=fn.error_msg)
             )
             return cls(fn, args, kwargs, return_type, is_method_call=is_method_call, validation_error=validation_error)
 

pixeltable/exprs/row_builder.py

@@ -209,7 +209,7 @@ class RowBuilder:
                 # this is input and therefore doesn't depend on other exprs
                 continue
             # error properties don't have exceptions themselves
-            if isinstance(expr, ColumnPropertyRef) and expr.is_error_prop():
+            if isinstance(expr, ColumnPropertyRef) and expr.is_cellmd_prop():
                 continue
             dependency_idxs = [d.slot_idx for d in expr.dependencies()]
             self.dependencies[expr.slot_idx, dependency_idxs] = True
@@ -444,6 +444,8 @@ class RowBuilder:
         Return tuple[list of row values in `self.table_columns` order, # of exceptions]
             This excludes system columns.
         """
+        from pixeltable.exprs.column_property_ref import ColumnPropertyRef
+
         num_excs = 0
         table_row: list[Any] = list(pk)
         for info in self.table_columns:
@@ -454,9 +456,9 @@ class RowBuilder:
                 if cols_with_excs is not None:
                     cols_with_excs.add(col.id)
                 table_row.append(None)
-                if col.records_errors:
-                    # exceptions get stored in the errortype/-msg columns
-                    table_row.extend((type(exc).__name__, str(exc)))
+                if col.stores_cellmd:
+                    # exceptions get stored in the errortype/-msg properties of the cellmd column
+                    table_row.append(ColumnPropertyRef.create_cellmd_exc(exc))
             else:
                 if col.col_type.is_image_type() and data_row.file_urls[slot_idx] is None:
                     # we have yet to store this image
@@ -464,8 +466,8 @@ class RowBuilder:
                     data_row.flush_img(slot_idx, filepath)
                 val = data_row.get_stored_val(slot_idx, col.get_sa_col_type())
                 table_row.append(val)
-                if col.records_errors:
-                    table_row.extend((None, None))
+                if col.stores_cellmd:
+                    table_row.append(None)  # placeholder for cellmd column
 
         return table_row, num_excs
 
@@ -483,8 +485,7 @@ class RowBuilder:
             if col.col.col_type.is_media_type():
                 media_cols[len(store_col_names)] = col.col
             store_col_names.append(col.col.store_name())
-            if col.col.records_errors:
-                store_col_names.append(col.col.errortype_store_name())
-                store_col_names.append(col.col.errormsg_store_name())
+            if col.col.stores_cellmd:
+                store_col_names.append(col.col.cellmd_store_name())
 
         return store_col_names, media_cols

pixeltable/exprs/rowid_ref.py

@@ -105,10 +105,6 @@ class RowidRef(Expr):
         assert self.rowid_component_idx <= len(rowid_cols), (
             f'{self.rowid_component_idx} not consistent with {rowid_cols}'
         )
-        # _logger.debug(
-        #     f'RowidRef.sql_expr: tbl={tbl.id}{tbl.effective_version} sa_tbl={id(tbl.store_tbl.sa_tbl):x} '
-        #     f'tv={id(tbl):x}'
-        # )
         return rowid_cols[self.rowid_component_idx]
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:

pixeltable/func/function.py

@@ -504,12 +504,12 @@ class Function(ABC):
 
 class InvalidFunction(Function):
     fn_dict: dict[str, Any]
-    errormsg: str
+    error_msg: str
 
-    def __init__(self, self_path: str, fn_dict: dict[str, Any], errormsg: str):
+    def __init__(self, self_path: str, fn_dict: dict[str, Any], error_msg: str):
         super().__init__([], self_path)
         self.fn_dict = fn_dict
-        self.errormsg = errormsg
+        self.error_msg = error_msg
 
     def _as_dict(self) -> dict:
         """