pixeltable 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

pixeltable/env.py

@@ -8,13 +8,15 @@ import importlib.util
 import inspect
 import logging
 import os
+import subprocess
 import sys
 import threading
 import uuid
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Callable, Optional, Dict, Any, List, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, Optional
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import pixeltable_pgserver
 import sqlalchemy as sql
@@ -37,6 +39,35 @@ class Env:
     _instance: Optional[Env] = None
     _log_fmt_str = '%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d: %(message)s'
 
+    _home: Optional[Path]
+    _media_dir: Optional[Path]
+    _file_cache_dir: Optional[Path]  # cached media files with external URL
+    _dataset_cache_dir: Optional[Path]  # cached datasets (eg, pytorch or COCO)
+    _log_dir: Optional[Path]  # log files
+    _tmp_dir: Optional[Path]  # any tmp files
+    _sa_engine: Optional[sql.engine.base.Engine]
+    _pgdata_dir: Optional[Path]
+    _db_name: Optional[str]
+    _db_server: Optional[pixeltable_pgserver.PostgresServer]
+    _db_url: Optional[str]
+    _default_time_zone: Optional[ZoneInfo]
+
+    # info about optional packages that are utilized by some parts of the code
+    __optional_packages: dict[str, PackageInfo]
+
+    _spacy_nlp: Optional[spacy.Language]
+    _httpd: Optional[http.server.HTTPServer]
+    _http_address: Optional[str]
+    _logger: logging.Logger
+    _default_log_level: int
+    _logfilename: Optional[str]
+    _log_to_stdout: bool
+    _module_log_level: dict[str, int]  # module name -> log level
+    _config_file: Optional[Path]
+    _config: Optional[dict[str, Any]]
+    _stdout_handler: logging.StreamHandler
+    _initialized: bool
+
     @classmethod
     def get(cls) -> Env:
         if cls._instance is None:
@@ -51,24 +82,23 @@ class Env:
         cls._instance = env
 
     def __init__(self):
-        self._home: Optional[Path] = None
-        self._media_dir: Optional[Path] = None  # computed media files
-        self._file_cache_dir: Optional[Path] = None  # cached media files with external URL
-        self._dataset_cache_dir: Optional[Path] = None  # cached datasets (eg, pytorch or COCO)
-        self._log_dir: Optional[Path] = None  # log files
-        self._tmp_dir: Optional[Path] = None  # any tmp files
-        self._sa_engine: Optional[sql.engine.base.Engine] = None
-        self._pgdata_dir: Optional[Path] = None
-        self._db_name: Optional[str] = None
-        self._db_server: Optional[pixeltable_pgserver.PostgresServer] = None
-        self._db_url: Optional[str] = None
-
-        # info about installed packages that are utilized by some parts of the code;
-        # package name -> version; version == []: package is installed, but we haven't determined the version yet
-        self._installed_packages: Dict[str, Optional[List[int]]] = {}
-        self._spacy_nlp: Optional[spacy.Language] = None
-        self._httpd: Optional[http.server.HTTPServer] = None
-        self._http_address: Optional[str] = None
+        self._home = None
+        self._media_dir = None  # computed media files
+        self._file_cache_dir = None  # cached media files with external URL
+        self._dataset_cache_dir = None  # cached datasets (eg, pytorch or COCO)
+        self._log_dir = None  # log files
+        self._tmp_dir = None  # any tmp files
+        self._sa_engine = None
+        self._pgdata_dir = None
+        self._db_name = None
+        self._db_server = None
+        self._db_url = None
+        self._default_time_zone = None
+
+        self.__optional_packages = {}
+        self._spacy_nlp = None
+        self._httpd = None
+        self._http_address = None
 
         # logging-related state
         self._logger = logging.getLogger('pixeltable')
@@ -76,13 +106,12 @@ class Env:
         self._logger.propagate = False
         self._logger.addFilter(self._log_filter)
         self._default_log_level = logging.INFO
-        self._logfilename: Optional[str] = None
+        self._logfilename = None
         self._log_to_stdout = False
-        self._module_log_level: Dict[str, int] = {}  # module name -> log level
+        self._module_log_level = {}  # module name -> log level
 
-        # config
-        self._config_file: Optional[Path] = None
-        self._config: Optional[Dict[str, Any]] = None
+        self._config_file = None
+        self._config = None
 
         # create logging handler to also log to stdout
         self._stdout_handler = logging.StreamHandler(stream=sys.stdout)
@@ -103,6 +132,19 @@ class Env:
         assert self._http_address is not None
         return self._http_address
 
+    @property
+    def default_time_zone(self) -> Optional[ZoneInfo]:
+        return self._default_time_zone
+
+    @default_time_zone.setter
+    def default_time_zone(self, tz: Optional[ZoneInfo]) -> None:
+        """
+        This is not a publicly visible setter; it is only for testing purposes.
+        """
+        tz_name = None if tz is None else tz.key
+        self.engine.dispose()
+        self._create_engine(time_zone_name=tz_name)
+
     def configure_logging(
         self,
         *,
@@ -158,7 +200,8 @@ class Env:
             self._module_log_level[module] = level
 
     def is_installed_package(self, package_name: str) -> bool:
-        return self._installed_packages[package_name] is not None
+        assert package_name in self.__optional_packages
+        return self.__optional_packages[package_name].is_installed
 
     def _log_filter(self, record: logging.LogRecord) -> bool:
         if record.name == 'pixeltable':
@@ -270,20 +313,35 @@ class Env:
         self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=None)
         self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
 
-        if reinit_db:
-            if self._store_db_exists():
-                self._drop_store_db()
+        tz_name = os.environ.get('PXT_TIME_ZONE', self._config.get('pxt_time_zone', None))
+        if tz_name is not None:
+            # Validate tzname
+            if not isinstance(tz_name, str):
+                self._logger.error(f'Invalid time zone specified in configuration.')
+            else:
+                try:
+                    _ = ZoneInfo(tz_name)
+                except ZoneInfoNotFoundError:
+                    self._logger.error(f'Invalid time zone specified in configuration: {tz_name}')
+
+        if reinit_db and self._store_db_exists():
+            self._drop_store_db()
 
-        if not self._store_db_exists():
-            self._logger.info(f'creating database at {self.db_url}')
+        create_db = not self._store_db_exists()
+
+        if create_db:
+            self._logger.info(f'creating database at: {self.db_url}')
             self._create_store_db()
-            self._create_engine(echo=echo)
+        else:
+            self._logger.info(f'found database at: {self.db_url}')
+
+        # Create the SQLAlchemy engine. This will also set the default time zone.
+        self._create_engine(time_zone_name=tz_name, echo=echo)
+
+        if create_db:
             from pixeltable.metadata import schema
             schema.Base.metadata.create_all(self._sa_engine)
             metadata.create_system_info(self._sa_engine)
-        else:
-            self._logger.info(f'found database {self.db_url}')
-            self._create_engine(echo=echo)
 
         print(f'Connected to Pixeltable database at: {self.db_url}')
 
@@ -291,8 +349,21 @@ class Env:
         self._set_up_runtime()
         self.log_to_stdout(False)
 
-    def _create_engine(self, echo: bool = False) -> None:
-        self._sa_engine = sql.create_engine(self.db_url, echo=echo, future=True, isolation_level='AUTOCOMMIT')
+    def _create_engine(self, time_zone_name: Optional[str], echo: bool = False) -> None:
+        connect_args = {} if time_zone_name is None else {'options': f'-c timezone={time_zone_name}'}
+        self._sa_engine = sql.create_engine(
+            self.db_url,
+            echo=echo,
+            future=True,
+            isolation_level='AUTOCOMMIT',
+            connect_args=connect_args,
+        )
+        self._logger.info(f'Created SQLAlchemy engine at: {self.db_url}')
+        with self.engine.begin() as conn:
+            tz_name = conn.execute(sql.text('SHOW TIME ZONE')).scalar()
+            assert isinstance(tz_name, str)
+            self._logger.info(f'Database time zone is now: {tz_name}')
+            self._default_time_zone = ZoneInfo(tz_name)
 
     def _store_db_exists(self) -> bool:
         assert self._db_name is not None
@@ -308,7 +379,6 @@ class Env:
         finally:
             engine.dispose()
 
-
     def _create_store_db(self) -> None:
         assert self._db_name is not None
         # create the db
@@ -416,63 +486,104 @@ class Env:
     def _set_up_runtime(self) -> None:
         """Check for and start runtime services"""
         self._start_web_server()
-        self._check_installed_packages()
-
-    def _check_installed_packages(self) -> None:
-        def check(package: str) -> None:
-            if importlib.util.find_spec(package) is not None:
-                self._installed_packages[package] = []
-            else:
-                self._installed_packages[package] = None
-
-        check('toml')
-        check('datasets')
-        check('torch')
-        check('torchvision')
-        check('transformers')
-        check('sentence_transformers')
-        check('whisper')
-        check('yolox')
-        check('whisperx')
-        check('boto3')
-        check('fitz')  # pymupdf
-        check('pyarrow')
-        check('spacy')  # TODO: deal with en-core-web-sm
+        self.__register_packages()
         if self.is_installed_package('spacy'):
-            import spacy
-
-            self._spacy_nlp = spacy.load('en_core_web_sm')
-        check('tiktoken')
-        check('openai')
-        check('anthropic')
-        check('together')
-        check('fireworks')
-        check('label_studio_sdk')
-        check('openpyxl')
-
-    def require_package(self, package: str, min_version: Optional[List[int]] = None) -> None:
-        assert package in self._installed_packages
-        if self._installed_packages[package] is None:
-            raise excs.Error(f'Package {package} is not installed')
+            self.__init_spacy()
+
+    def __register_packages(self) -> None:
+        """Declare optional packages that are utilized by some parts of the code."""
+        self.__register_package('anthropic')
+        self.__register_package('boto3')
+        self.__register_package('datasets')
+        self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
+        self.__register_package('mistralai')
+        self.__register_package('mistune')
+        self.__register_package('openai')
+        self.__register_package('openpyxl')
+        self.__register_package('pyarrow')
+        self.__register_package('sentence_transformers', library_name='sentence-transformers')
+        self.__register_package('spacy')
+        self.__register_package('tiktoken')
+        self.__register_package('together')
+        self.__register_package('toml')
+        self.__register_package('torch')
+        self.__register_package('torchvision')
+        self.__register_package('transformers')
+        self.__register_package('whisper', library_name='openai-whisper')
+        self.__register_package('whisperx')
+        self.__register_package('yolox', library_name='git+https://github.com/Megvii-BaseDetection/YOLOX@ac58e0a')
+
+    def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+        self.__optional_packages[package_name] = PackageInfo(
+            is_installed=importlib.util.find_spec(package_name) is not None,
+            library_name=library_name or package_name  # defaults to package_name unless specified otherwise
+        )
+
+    def require_package(self, package_name: str, min_version: Optional[list[int]] = None) -> None:
+        """
+        Checks whether the specified optional package is available. If not, raises an exception
+        with an error message informing the user how to install it.
+        """
+        assert package_name in self.__optional_packages
+        package_info = self.__optional_packages[package_name]
+
+        if not package_info.is_installed:
+            # Check again whether the package has been installed.
+            # We do this so that if a user gets an "optional library not found" error message, they can
+            # `pip install` the library and re-run the Pixeltable operation without having to restart
+            # their Python session.
+            package_info.is_installed = importlib.util.find_spec(package_name) is not None
+            if not package_info.is_installed:
+                # Still not found.
+                raise excs.Error(
+                    f'This feature requires the `{package_name}` package. To install it, run: `pip install -U {package_info.library_name}`'
+                )
+
         if min_version is None:
             return
 
         # check whether we have a version >= the required one
-        if not self._installed_packages[package]:
-            m = importlib.import_module(package)
-            module_version = [int(x) for x in m.__version__.split('.')]
-            self._installed_packages[package] = module_version
-        installed_version = self._installed_packages[package]
-        if len(min_version) < len(installed_version):
-            normalized_min_version = min_version + [0] * (len(installed_version) - len(min_version))
-        if any([a < b for a, b in zip(installed_version, normalized_min_version)]):
+        if package_info.version is None:
+            module = importlib.import_module(package_name)
+            package_info.version = [int(x) for x in module.__version__.split('.')]
+
+        if min_version > package_info.version:
             raise excs.Error(
-                (
-                    f'The installed version of package {package} is {".".join(str(v) for v in installed_version)}, '
-                    f'but version  >={".".join(str(v) for v in min_version)} is required'
-                )
+                f'The installed version of package `{package_name}` is {".".join(str(v) for v in package_info.version)}, '
+                f'but version >={".".join(str(v) for v in min_version)} is required. '
+                f'To fix this, run: `pip install -U {package_info.library_name}`'
             )
 
+    def __init_spacy(self) -> None:
+        """
+        spaCy relies on a pip-installed model to operate. In order to avoid requiring the model as a separate
+        dependency, we install it programmatically here. This should cause no problems, since the model packages
+        have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
+        """
+        import spacy
+        from spacy.cli.download import get_model_filename
+        spacy_model = 'en_core_web_sm'
+        spacy_model_version = '3.7.1'
+        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
+        url = f'{spacy.about.__download_url__}/{filename}'
+        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
+        # a problem, because the model have been installed on a previous attempt.
+        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
+        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
+        if ret.returncode != 0:
+            self._logger.warn(f'pip install failed for spaCy model: {filename}')
+        try:
+            self._logger.info(f'Loading spaCy model: {spacy_model}')
+            self._spacy_nlp = spacy.load(spacy_model)
+        except Exception as exc:
+            self._logger.warn(f'Failed to load spaCy model: {spacy_model}', exc_info=exc)
+            warnings.warn(
+                f"Failed to load spaCy model '{spacy_model}'. spaCy features will not be available.",
+                excs.PixeltableWarning
+            )
+            self.__optional_packages['spacy'].is_installed = False
+
     def num_tmp_files(self) -> int:
         return len(glob.glob(f'{self._tmp_dir}/*'))
 
@@ -511,6 +622,7 @@ class Env:
 
     @property
     def spacy_nlp(self) -> spacy.Language:
+        Env.get().require_package('spacy')
         assert self._spacy_nlp is not None
         return self._spacy_nlp
 
@@ -556,3 +668,10 @@ class ApiClient:
     init_fn: Callable
     param_names: list[str]
     client_obj: Optional[Any] = None
+
+
+@dataclass
+class PackageInfo:
+    is_installed: bool
+    library_name: str  # pypi library name (may be different from package name)
+    version: Optional[list[int]] = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")

pixeltable/exceptions.py

@@ -14,4 +14,8 @@ class ExprEvalError(Exception):
     exc: Exception
     exc_tb: TracebackType
     input_vals: List[Any]
-    row_num: int
+    row_num: int
+
+
+class PixeltableWarning(Warning):
+    pass

pixeltable/exec/aggregation_node.py

@@ -21,8 +21,9 @@ class AggregationNode(ExecNode):
         self.input = input
         self.group_by = group_by
         self.input_exprs = list(input_exprs)
-        self.agg_fn_calls = agg_fn_calls
         self.agg_fn_eval_ctx = row_builder.create_eval_ctx(agg_fn_calls, exclude=input_exprs)
+        # we need to make sure to refer to the same exprs that RowBuilder.eval() will use
+        self.agg_fn_calls = self.agg_fn_eval_ctx.target_exprs
         self.output_batch = DataRowBatch(tbl, row_builder, 0)
 
     def _reset_agg_state(self, row_num: int) -> None:

pixeltable/exec/component_iteration_node.py

@@ -19,12 +19,12 @@ class ComponentIterationNode(ExecNode):
         super().__init__(input.row_builder, [], [], input)
         self.view = view
         iterator_args = [view.iterator_args.copy()]
-        self.row_builder.substitute_exprs(iterator_args)
+        self.row_builder.set_slot_idxs(iterator_args)
         self.iterator_args = iterator_args[0]
         assert isinstance(self.iterator_args, exprs.InlineDict)
         self.iterator_args_ctx = self.row_builder.create_eval_ctx([self.iterator_args])
         self.iterator_output_schema, self.unstored_column_names = \
-            self.view.iterator_cls.output_schema(**self.iterator_args.to_dict())
+            self.view.iterator_cls.output_schema(**self.iterator_args.to_kwargs())
         self.iterator_output_fields = list(self.iterator_output_schema.keys())
         self.iterator_output_cols = \
             {field_name: self.view.cols_by_name[field_name] for field_name in self.iterator_output_fields}

pixeltable/exec/sql_node.py

@@ -18,7 +18,7 @@ class SqlNode(ExecNode):
 
     def __init__(
             self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
-            select_list: Iterable[exprs.Expr], set_pk: bool = False
+            select_list: Iterable[exprs.Expr], sql_elements: exprs.SqlElementCache, set_pk: bool = False
     ):
         """
         Initialize self.stmt with expressions derived from select_list.
@@ -35,8 +35,9 @@ class SqlNode(ExecNode):
         self.sql_exprs = exprs.ExprSet(select_list)
         # unstored iter columns: we also need to retrieve whatever is needed to materialize the iter args
         for iter_arg in row_builder.unstored_iter_args.values():
-            sql_subexprs = iter_arg.subexprs(filter=lambda e: e.sql_expr() is not None, traverse_matches=False)
-            [self.sql_exprs.append(e) for e in sql_subexprs]
+            sql_subexprs = iter_arg.subexprs(filter=sql_elements.contains, traverse_matches=False)
+            for e in sql_subexprs:
+                self.sql_exprs.add(e)
         super().__init__(row_builder, self.sql_exprs, [], None)  # we materialize self.sql_exprs
 
         # change rowid refs against a base table to rowid refs against the target table, so that we minimize
@@ -44,7 +45,7 @@ class SqlNode(ExecNode):
         for rowid_ref in [e for e in self.sql_exprs if isinstance(e, exprs.RowidRef)]:
             rowid_ref.set_tbl(tbl)
 
-        sql_select_list = [e.sql_expr() for e in self.sql_exprs]
+        sql_select_list = [sql_elements.get(e) for e in self.sql_exprs]
         assert len(sql_select_list) == len(self.sql_exprs)
         assert all(e is not None for e in sql_select_list)
         self.set_pk = set_pk
@@ -204,7 +205,8 @@ class SqlScanNode(SqlNode):
             set_pk: if True, sets the primary for each DataRow
             exact_version_only: tables for which we only want to see rows created at the current version
         """
-        super().__init__(tbl, row_builder, select_list, set_pk=set_pk)
+        sql_elements = exprs.SqlElementCache()
+        super().__init__(tbl, row_builder, select_list, sql_elements, set_pk=set_pk)
         # create Select stmt
         if order_by_items is None:
             order_by_items = []
@@ -230,10 +232,10 @@ class SqlScanNode(SqlNode):
             if isinstance(e, exprs.SimilarityExpr):
                 order_by_clause.append(e.as_order_by_clause(asc))
             else:
-                order_by_clause.append(e.sql_expr().desc() if not asc else e.sql_expr())
+                order_by_clause.append(sql_elements.get(e).desc() if not asc else sql_elements.get(e))
 
         if where_clause is not None:
-            sql_where_clause = where_clause.sql_expr()
+            sql_where_clause = sql_elements.get(where_clause)
             assert sql_where_clause is not None
             self.stmt = self.stmt.where(sql_where_clause)
         if len(order_by_clause) > 0:
@@ -272,7 +274,8 @@ class SqlLookupNode(SqlNode):
             sa_key_cols: list of key columns in the store table
             key_vals: list of key values to look up
         """
-        super().__init__(tbl, row_builder, select_list, set_pk=True)
+        sql_elements = exprs.SqlElementCache()
+        super().__init__(tbl, row_builder, select_list, sql_elements, set_pk=True)
         target = tbl.tbl_version  # the stored table we're scanning
         refd_tbl_ids = exprs.Expr.list_tbl_ids(self.sql_exprs)
         self.stmt = self.create_from_clause(tbl, self.stmt, refd_tbl_ids)

pixeltable/exprs/__init__.py

@@ -9,8 +9,7 @@ from .expr import Expr
 from .expr_set import ExprSet
 from .function_call import FunctionCall
 from .in_predicate import InPredicate
-from .inline_array import InlineArray
-from .inline_dict import InlineDict
+from .inline_expr import InlineArray, InlineDict, InlineList
 from .is_null import IsNull
 from .json_mapper import JsonMapper
 from .json_path import RELATIVE_PATH_ROOT, JsonPath
@@ -20,5 +19,6 @@ from .object_ref import ObjectRef
 from .row_builder import RowBuilder, ColumnSlotIdx, ExecProfile
 from .rowid_ref import RowidRef
 from .similarity_expr import SimilarityExpr
+from .sql_element_cache import SqlElementCache
 from .type_cast import TypeCast
 from .variable import Variable

pixeltable/exprs/arithmetic_expr.py

@@ -6,11 +6,11 @@ import sqlalchemy as sql
 
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
-
 from .data_row import DataRow
 from .expr import Expr
 from .globals import ArithmeticOperator
 from .row_builder import RowBuilder
+from .sql_element_cache import SqlElementCache
 
 
 class ArithmeticExpr(Expr):
@@ -54,10 +54,10 @@ class ArithmeticExpr(Expr):
     def _op2(self) -> Expr:
         return self.components[1]
 
-    def sql_expr(self) -> Optional[sql.ClauseElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
         assert self.col_type.is_int_type() or self.col_type.is_float_type() or self.col_type.is_json_type()
-        left = self._op1.sql_expr()
-        right = self._op2.sql_expr()
+        left = sql_elements.get(self._op1)
+        right = sql_elements.get(self._op2)
         if left is None or right is None:
             return None
         if self.operator == ArithmeticOperator.ADD:

pixeltable/exprs/array_slice.py

@@ -8,6 +8,7 @@ from .data_row import DataRow
 from .expr import Expr
 from .globals import print_slice
 from .row_builder import RowBuilder
+from .sql_element_cache import SqlElementCache
 
 
 class ArraySlice(Expr):
@@ -41,7 +42,7 @@ class ArraySlice(Expr):
     def _id_attrs(self) -> List[Tuple[str, Any]]:
         return super()._id_attrs() + [('index', self.index)]
 
-    def sql_expr(self) -> Optional[sql.ClauseElement]:
+    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
         return None
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:

pixeltable/exprs/column_property_ref.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
-from typing import Optional, List, Any, Dict, Tuple
+
 import enum
+from typing import Optional, List, Any, Dict, Tuple
 
 import sqlalchemy as sql
 
-from .expr import Expr
+import pixeltable.type_system as ts
 from .column_ref import ColumnRef
-from .row_builder import RowBuilder
 from .data_row import DataRow
-import pixeltable.catalog as catalog
-import pixeltable.type_system as ts
+from .expr import Expr
+from .row_builder import RowBuilder
+from .sql_element_cache import SqlElementCache
 
 
 class ColumnPropertyRef(Expr):
@@ -45,7 +46,7 @@ class ColumnPropertyRef(Expr):
     def __str__(self) -> str:
         return f'{self._col_ref}.{self.prop.name.lower()}'
 
-    def sql_expr(self) -> Optional[sql.ClauseElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
         if not self._col_ref.col.is_stored:
             return None
         if self.prop == self.Property.ERRORTYPE:
@@ -56,7 +57,7 @@ class ColumnPropertyRef(Expr):
             return self._col_ref.col.sa_errormsg_col
         if self.prop == self.Property.FILEURL:
             # the file url is stored as the column value
-            return self._col_ref.sql_expr()
+            return sql_elements.get(self._col_ref)
         return None
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
@@ -73,5 +74,6 @@ class ColumnPropertyRef(Expr):
     @classmethod
     def _from_dict(cls, d: Dict, components: List[Expr]) -> Expr:
         assert 'prop' in d
+        assert isinstance(components[0], ColumnRef)
         return cls(components[0], cls.Property(d['prop']))
 

pixeltable/exprs/column_ref.py

@@ -7,6 +7,7 @@ import sqlalchemy as sql
 from .expr import Expr
 from .data_row import DataRow
 from .row_builder import RowBuilder
+from .sql_element_cache import SqlElementCache
 import pixeltable.iterators as iters
 import pixeltable.exceptions as excs
 import pixeltable.catalog as catalog
@@ -92,7 +93,7 @@ class ColumnRef(Expr):
     def __repr__(self) -> str:
         return f'ColumnRef({self.col!r})'
 
-    def sql_expr(self) -> Optional[sql.ClauseElement]:
+    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
         return self.col.sa_col
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:

pixeltable/exprs/comparison.py

@@ -1,22 +1,25 @@
 from __future__ import annotations
 
-from datetime import datetime
 from typing import Optional, List, Any, Dict
 
 import sqlalchemy as sql
 
+import pixeltable.exceptions as excs
+import pixeltable.index as index
+import pixeltable.type_system as ts
 from .column_ref import ColumnRef
 from .data_row import DataRow
 from .expr import Expr
 from .globals import ComparisonOperator
 from .literal import Literal
 from .row_builder import RowBuilder
-import pixeltable.exceptions as excs
-import pixeltable.index as index
-import pixeltable.type_system as ts
+from .sql_element_cache import SqlElementCache
 
 
 class Comparison(Expr):
+    is_search_arg_comparison: bool
+    operator: ComparisonOperator
+
     def __init__(self, operator: ComparisonOperator, op1: Expr, op2: Expr):
         super().__init__(ts.BoolType())
         self.operator = operator
@@ -62,8 +65,8 @@ class Comparison(Expr):
     def _op2(self) -> Expr:
         return self.components[1]
 
-    def sql_expr(self) -> Optional[sql.ClauseElement]:
-        left = self._op1.sql_expr()
+    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ClauseElement]:
+        left = sql_elements.get(self._op1)
         if self.is_search_arg_comparison:
             # reference the index value column if there is an index and this is not a snapshot
             # (indices don't apply to snapshots)
@@ -76,7 +79,7 @@ class Comparison(Expr):
                 assert len(idx_info) == 1
                 left = idx_info[0].val_col.sa_col
 
-        right = self._op2.sql_expr()
+        right = sql_elements.get(self._op2)
         if left is None or right is None:
             return None