pixeltable 0.2.24__py3-none-any.whl → 0.3.0__py3-none-any.whl

pixeltable/env.py

@@ -8,6 +8,7 @@ import importlib.util
 import inspect
 import logging
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -275,6 +276,7 @@ class Env:
         if self._config.get_bool_value('hide_warnings'):
             # Disable more warnings
             warnings.simplefilter('ignore', category=UserWarning)
+            warnings.simplefilter('ignore', category=FutureWarning)
 
         # configure _logger to log to a file
         self._logfilename = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + '.log'
@@ -311,8 +313,12 @@ class Env:
         self._db_name = os.environ.get('PIXELTABLE_DB', 'pixeltable')
         self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(self._home / 'pgdata')))
 
-        # in pixeltable_pgserver.get_server(): cleanup_mode=None will leave db on for debugging purposes
-        self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=None)
+        # cleanup_mode=None will leave the postgres process running after Python exits
+        # cleanup_mode='stop' will terminate the postgres process when Python exits
+        # On Windows, we need cleanup_mode='stop' because child processes are killed automatically when the parent
+        # process (such as Terminal or VSCode) exits, potentially leaving it in an unusable state.
+        cleanup_mode = 'stop' if platform.system() == 'Windows' else None
+        self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=cleanup_mode)
         self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
 
         tz_name = self.config.get_string_value('time_zone')
@@ -357,7 +363,7 @@ class Env:
             self.db_url,
             echo=echo,
             future=True,
-            isolation_level='AUTOCOMMIT',
+            isolation_level='REPEATABLE READ',
             connect_args=connect_args,
         )
         self._logger.info(f'Created SQLAlchemy engine at: {self.db_url}')
@@ -496,6 +502,7 @@ class Env:
         self.__register_package('datasets')
         self.__register_package('fiftyone')
         self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('google.generativeai', library_name='google-generativeai')
         self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
         self.__register_package('llama_cpp', library_name='llama-cpp-python')
@@ -505,6 +512,7 @@ class Env:
         self.__register_package('openai')
         self.__register_package('openpyxl')
         self.__register_package('pyarrow')
+        self.__register_package('pydantic')
         self.__register_package('replicate')
         self.__register_package('sentencepiece')
         self.__register_package('sentence_transformers', library_name='sentence-transformers')
@@ -520,8 +528,14 @@ class Env:
         self.__register_package('yolox', library_name='git+https://github.com/Megvii-BaseDetection/YOLOX@ac58e0a')
 
     def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+        is_installed: bool
+        try:
+            is_installed = importlib.util.find_spec(package_name) is not None
+        except ModuleNotFoundError:
+            # This can happen if the parent of `package_name` is not installed.
+            is_installed = False
         self.__optional_packages[package_name] = PackageInfo(
-            is_installed=importlib.util.find_spec(package_name) is not None,
+            is_installed=is_installed,
             library_name=library_name or package_name  # defaults to package_name unless specified otherwise
         )
 
@@ -577,7 +591,7 @@ class Env:
         self._logger.info(f'Ensuring spaCy model is installed: {filename}')
         ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
         if ret.returncode != 0:
-            self._logger.warn(f'pip install failed for spaCy model: {filename}')
+            self._logger.warning(f'pip install failed for spaCy model: {filename}')
         try:
             self._logger.info(f'Loading spaCy model: {spacy_model}')
             self._spacy_nlp = spacy.load(spacy_model)

pixeltable/exec/__init__.py

@@ -7,4 +7,4 @@ from .exec_node import ExecNode
 from .expr_eval_node import ExprEvalNode
 from .in_memory_data_node import InMemoryDataNode
 from .row_update_node import RowUpdateNode
-from .sql_node import SqlLookupNode, SqlScanNode, SqlAggregationNode, SqlNode
+from .sql_node import SqlLookupNode, SqlScanNode, SqlAggregationNode, SqlNode, SqlJoinNode

pixeltable/exec/exec_node.py

@@ -1,15 +1,13 @@
 from __future__ import annotations
 
 import abc
-from typing import TYPE_CHECKING, Iterable, Iterator, Optional
+from typing import TYPE_CHECKING, Iterable, Iterator, Optional, TypeVar
 
 import pixeltable.exprs as exprs
 
 from .data_row_batch import DataRowBatch
 from .exec_context import ExecContext
 
-if TYPE_CHECKING:
-    from pixeltable import exec
 
 class ExecNode(abc.ABC):
     """Base class of all execution nodes"""
@@ -77,12 +75,13 @@ class ExecNode(abc.ABC):
     def _close(self) -> None:
         pass
 
-    def get_sql_node(self) -> Optional['exec.SqlNode']:
-        from .sql_node import SqlNode
-        if isinstance(self, SqlNode):
+    T = TypeVar('T', bound='ExecNode')
+
+    def get_node(self, node_class: type[T]) -> Optional[T]:
+        if isinstance(self, node_class):
             return self
         if self.input is not None:
-            return self.input.get_sql_node()
+            return self.input.get_node(node_class)
         return None
 
     def set_limit(self, limit: int) -> None:

pixeltable/exec/expr_eval_node.py

@@ -208,7 +208,7 @@ class ExprEvalNode(ExecNode):
                         }
                         start_ts = time.perf_counter()
                         assert isinstance(fn_call.fn, CallableFunction)
-                        result_batch = fn_call.fn.exec_batch(*call_args, **call_kwargs)
+                        result_batch = fn_call.fn.exec_batch(call_args, call_kwargs)
                         self.ctx.profile.eval_time[fn_call.slot_idx] += time.perf_counter() - start_ts
                         self.ctx.profile.eval_count[fn_call.slot_idx] += num_ext_batch_rows
 

pixeltable/exec/sql_node.py

@@ -1,17 +1,19 @@
 import logging
 import warnings
 from decimal import Decimal
-from typing import Iterable, Iterator, NamedTuple, Optional
+from typing import Iterable, Iterator, NamedTuple, Optional, TYPE_CHECKING, Sequence
 from uuid import UUID
 
 import sqlalchemy as sql
 
 import pixeltable.catalog as catalog
 import pixeltable.exprs as exprs
-
 from .data_row_batch import DataRowBatch
 from .exec_node import ExecNode
 
+if TYPE_CHECKING:
+    import pixeltable.plan
+
 _logger = logging.getLogger('pixeltable')
 
 
@@ -67,12 +69,17 @@ class SqlNode(ExecNode):
     select_list: exprs.ExprSet
     set_pk: bool
     num_pk_cols: int
-    filter: Optional[exprs.Expr]
-    filter_eval_ctx: Optional[exprs.RowBuilder.EvalCtx]
+    py_filter: Optional[exprs.Expr]  # a predicate that can only be run in Python
+    py_filter_eval_ctx: Optional[exprs.RowBuilder.EvalCtx]
     cte: Optional[sql.CTE]
     sql_elements: exprs.SqlElementCache
-    limit: Optional[int]
+
+    # where_clause/-_element: allow subclass to set one or the other (but not both)
+    where_clause: Optional[exprs.Expr]
+    where_clause_element: Optional[sql.ColumnElement]
+
     order_by_clause: OrderByClause
+    limit: Optional[int]
 
     def __init__(
             self, tbl: Optional[catalog.TableVersionPath], row_builder: exprs.RowBuilder,
@@ -89,6 +96,7 @@ class SqlNode(ExecNode):
         # create Select stmt
         self.sql_elements = sql_elements
         self.tbl = tbl
+        assert all(not isinstance(e, exprs.Literal) for e in select_list)  # we're never asked to materialize literals
         self.select_list = exprs.ExprSet(select_list)
         # unstored iter columns: we also need to retrieve whatever is needed to materialize the iter args
         for iter_arg in row_builder.unstored_iter_args.values():
@@ -112,10 +120,12 @@ class SqlNode(ExecNode):
         # additional state
         self.result_cursor = None
         # the filter is provided by the subclass
-        self.filter = None
-        self.filter_eval_ctx = None
+        self.py_filter = None
+        self.py_filter_eval_ctx = None
         self.cte = None
         self.limit = None
+        self.where_clause = None
+        self.where_clause_element = None
         self.order_by_clause = []
 
     def _create_stmt(self) -> sql.Select:
@@ -124,9 +134,16 @@ class SqlNode(ExecNode):
         assert self.sql_elements.contains_all(self.select_list)
         sql_select_list = [self.sql_elements.get(e) for e in self.select_list]
         if self.set_pk:
+            assert self.tbl is not None
             sql_select_list += self.tbl.tbl_version.store_tbl.pk_columns()
         stmt = sql.select(*sql_select_list)
 
+        where_clause_element = (
+            self.sql_elements.get(self.where_clause) if self.where_clause is not None else self.where_clause_element
+        )
+        if where_clause_element is not None:
+            stmt = stmt.where(where_clause_element)
+
         order_by_clause: list[sql.ColumnElement] = []
         for e, asc in self.order_by_clause:
             if isinstance(e, exprs.SimilarityExpr):
@@ -135,7 +152,7 @@ class SqlNode(ExecNode):
                 order_by_clause.append(self.sql_elements.get(e).desc() if asc is False else self.sql_elements.get(e))
         stmt = stmt.order_by(*order_by_clause)
 
-        if self.filter is None and self.limit is not None:
+        if self.py_filter is None and self.limit is not None:
             # if we don't have a Python filter, we can apply the limit to stmt
             stmt = stmt.limit(self.limit)
 
@@ -151,7 +168,7 @@ class SqlNode(ExecNode):
         Returns:
             (CTE, dict from Expr to output column)
         """
-        if self.filter is not None:
+        if self.py_filter is not None:
             # the filter needs to run in Python
             return None
         self.set_pk = False  # we don't need the PK if we use this SqlNode as a CTE
@@ -215,8 +232,17 @@ class SqlNode(ExecNode):
             prev_tbl = tbl
         return stmt
 
-    def add_order_by(self, ordering: OrderByClause) -> None:
-        """Add Order By clause to stmt"""
+    def set_where(self, where_clause: exprs.Expr) -> None:
+        assert self.where_clause_element is None
+        self.where_clause = where_clause
+
+    def set_py_filter(self, py_filter: exprs.Expr) -> None:
+        assert self.py_filter is None
+        self.py_filter = py_filter
+        self.py_filter_eval_ctx = self.row_builder.create_eval_ctx([py_filter], exclude=self.select_list)
+
+    def set_order_by(self, ordering: OrderByClause) -> None:
+        """Add Order By clause"""
         if self.tbl is not None:
             # change rowid refs against a base table to rowid refs against the target table, so that we minimize
             # the number of tables that need to be joined to the target table
@@ -236,7 +262,7 @@ class SqlNode(ExecNode):
             explain_str = '\n'.join([str(row) for row in explain_result])
             _logger.debug(f'SqlScanNode explain:\n{explain_str}')
         except Exception as e:
-            _logger.warning(f'EXPLAIN failed')
+            _logger.warning(f'EXPLAIN failed with error: {e}')
 
     def __iter__(self) -> Iterator[DataRowBatch]:
         # run the query; do this here rather than in _open(), exceptions are only expected during iteration
@@ -280,10 +306,10 @@ class SqlNode(ExecNode):
                 else:
                     output_row[slot_idx] = sql_row[i]
 
-            if self.filter is not None:
+            if self.py_filter is not None:
                 # evaluate filter
-                self.row_builder.eval(output_row, self.filter_eval_ctx, profile=self.ctx.profile)
-            if self.filter is not None and not output_row[self.filter.slot_idx]:
+                self.row_builder.eval(output_row, self.py_filter_eval_ctx, profile=self.ctx.profile)
+            if self.py_filter is not None and not output_row[self.py_filter.slot_idx]:
                 # we re-use this row for the next sql row since it didn't pass the filter
                 output_row = output_batch.pop_row()
                 output_row.clear()
@@ -315,21 +341,16 @@ class SqlScanNode(SqlNode):
 
     Supports filtering and ordering.
     """
-    where_clause: Optional[exprs.Expr]
     exact_version_only: list[catalog.TableVersion]
 
     def __init__(
-            self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
-            select_list: Iterable[exprs.Expr],
-            where_clause: Optional[exprs.Expr] = None, filter: Optional[exprs.Expr] = None,
-            set_pk: bool = False, exact_version_only: Optional[list[catalog.TableVersion]] = None
+        self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
+        select_list: Iterable[exprs.Expr],
+        set_pk: bool = False, exact_version_only: Optional[list[catalog.TableVersion]] = None
     ):
         """
         Args:
             select_list: output of the query
-            sql_where_clause: SQL Where clause
-            filter: additional Where-clause predicate that can't be evaluated via SQL
-            limit: max number of rows to return: 0 = no limit
             set_pk: if True, sets the primary for each DataRow
             exact_version_only: tables for which we only want to see rows created at the current version
         """
@@ -338,12 +359,7 @@ class SqlScanNode(SqlNode):
         # create Select stmt
         if exact_version_only is None:
             exact_version_only = []
-        target = tbl.tbl_version  # the stored table we're scanning
-        self.filter = filter
-        self.filter_eval_ctx = \
-            row_builder.create_eval_ctx([filter], exclude=select_list) if filter is not None else None
 
-        self.where_clause = where_clause
         self.exact_version_only = exact_version_only
 
     def _create_stmt(self) -> sql.Select:
@@ -352,12 +368,6 @@ class SqlScanNode(SqlNode):
         refd_tbl_ids = exprs.Expr.all_tbl_ids(self.select_list) | where_clause_tbl_ids | self._ordering_tbl_ids()
         stmt = self.create_from_clause(
             self.tbl, stmt, refd_tbl_ids, exact_version_only={t.id for t in self.exact_version_only})
-
-        if self.where_clause is not None:
-            sql_where_clause = self.sql_elements.get(self.where_clause)
-            assert sql_where_clause is not None
-            stmt = stmt.where(sql_where_clause)
-
         return stmt
 
 
@@ -366,11 +376,9 @@ class SqlLookupNode(SqlNode):
     Materializes data from the store via a Select stmt with a WHERE clause that matches a list of key values
     """
 
-    where_clause: sql.ColumnElement
-
     def __init__(
-            self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
-            select_list: Iterable[exprs.Expr], sa_key_cols: list[sql.Column], key_vals: list[tuple],
+        self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
+        select_list: Iterable[exprs.Expr], sa_key_cols: list[sql.Column], key_vals: list[tuple],
     ):
         """
         Args:
@@ -381,15 +389,15 @@ class SqlLookupNode(SqlNode):
         sql_elements = exprs.SqlElementCache()
         super().__init__(tbl, row_builder, select_list, sql_elements, set_pk=True)
         # Where clause: (key-col-1, key-col-2, ...) IN ((val-1, val-2, ...), ...)
-        self.where_clause = sql.tuple_(*sa_key_cols).in_(key_vals)
+        self.where_clause_element = sql.tuple_(*sa_key_cols).in_(key_vals)
 
     def _create_stmt(self) -> sql.Select:
         stmt = super()._create_stmt()
         refd_tbl_ids = exprs.Expr.all_tbl_ids(self.select_list) | self._ordering_tbl_ids()
         stmt = self.create_from_clause(self.tbl, stmt, refd_tbl_ids)
-        stmt = stmt.where(self.where_clause)
         return stmt
 
+
 class SqlAggregationNode(SqlNode):
     """
     Materializes data from the store via a Select stmt with a WHERE clause that matches a list of key values
@@ -398,11 +406,11 @@ class SqlAggregationNode(SqlNode):
     group_by_items: Optional[list[exprs.Expr]]
 
     def __init__(
-            self, row_builder: exprs.RowBuilder,
-            input: SqlNode,
-            select_list: Iterable[exprs.Expr],
-            group_by_items: Optional[list[exprs.Expr]] = None,
-            limit: Optional[int] = None, exact_version_only: Optional[list[catalog.TableVersion]] = None
+        self, row_builder: exprs.RowBuilder,
+        input: SqlNode,
+        select_list: Iterable[exprs.Expr],
+        group_by_items: Optional[list[exprs.Expr]] = None,
+        limit: Optional[int] = None, exact_version_only: Optional[list[catalog.TableVersion]] = None
     ):
         """
         Args:
@@ -422,3 +430,42 @@ class SqlAggregationNode(SqlNode):
             assert all(e is not None for e in sql_group_by_items)
             stmt = stmt.group_by(*sql_group_by_items)
         return stmt
+
+
+class SqlJoinNode(SqlNode):
+    """
+    Materializes data from the store via a Select ... From ... that contains joins
+    """
+    input_ctes: list[sql.CTE]
+    join_clauses: list['pixeltable.plan.JoinClause']
+
+    def __init__(
+        self, row_builder: exprs.RowBuilder,
+        inputs: Sequence[SqlNode], join_clauses: list['pixeltable.plan.JoinClause'], select_list: Iterable[exprs.Expr]
+    ):
+        assert len(inputs) > 1
+        assert len(inputs) == len(join_clauses) + 1
+        self.input_ctes = []
+        self.join_clauses = join_clauses
+        sql_elements = exprs.SqlElementCache()
+        for input_node in inputs:
+            input_cte, input_col_map = input_node.to_cte()
+            self.input_ctes.append(input_cte)
+            sql_elements.extend(input_col_map)
+        super().__init__(None, row_builder, select_list, sql_elements)
+
+    def _create_stmt(self) -> sql.Select:
+        from pixeltable import plan
+        stmt = super()._create_stmt()
+        stmt = stmt.select_from(self.input_ctes[0])
+        for i in range(len(self.join_clauses)):
+            join_clause = self.join_clauses[i]
+            on_clause = (
+                self.sql_elements.get(join_clause.join_predicate) if join_clause.join_type != plan.JoinType.CROSS
+                else sql.sql.expression.literal(True)
+            )
+            is_outer = join_clause.join_type == plan.JoinType.LEFT or join_clause.join_type == plan.JoinType.FULL_OUTER
+            stmt = stmt.join(
+                self.input_ctes[i + 1], onclause=on_clause, isouter=is_outer,
+                full=join_clause == plan.JoinType.FULL_OUTER)
+        return stmt

pixeltable/exprs/__init__.py

@@ -23,3 +23,4 @@ from .similarity_expr import SimilarityExpr
 from .sql_element_cache import SqlElementCache
 from .type_cast import TypeCast
 from .variable import Variable
+from .globals import ComparisonOperator, LogicalOperator, ArithmeticOperator

pixeltable/exprs/arithmetic_expr.py

@@ -35,7 +35,7 @@ class ArithmeticExpr(Expr):
 
         self.id = self._create_id()
 
-    def __str__(self) -> str:
+    def __repr__(self) -> str:
         # add parentheses around operands that are ArithmeticExprs to express precedence
         op1_str = f'({self._op1})' if isinstance(self._op1, ArithmeticExpr) else str(self._op1)
         op2_str = f'({self._op2})' if isinstance(self._op2, ArithmeticExpr) else str(self._op2)

pixeltable/exprs/array_slice.py

@@ -23,7 +23,7 @@ class ArraySlice(Expr):
         self.index = index
         self.id = self._create_id()
 
-    def __str__(self) -> str:
+    def __repr__(self) -> str:
         index_strs: list[str] = []
         for el in self.index:
             if isinstance(el, int):

pixeltable/exprs/column_property_ref.py

@@ -46,7 +46,7 @@ class ColumnPropertyRef(Expr):
         assert isinstance(col_ref, ColumnRef)
         return col_ref
 
-    def __str__(self) -> str:
+    def __repr__(self) -> str:
         return f'{self._col_ref}.{self.prop.name.lower()}'
 
     def is_error_prop(self) -> bool:

pixeltable/exprs/column_ref.py

@@ -5,10 +5,12 @@ from uuid import UUID
 
 import sqlalchemy as sql
 
+import pixeltable as pxt
 import pixeltable.catalog as catalog
 import pixeltable.exceptions as excs
 import pixeltable.iterators as iters
 
+from ..utils.description_helper import DescriptionHelper
 from .data_row import DataRow
 from .expr import Expr
 from .row_builder import RowBuilder
@@ -126,6 +128,22 @@ class ColumnRef(Expr):
     def _equals(self, other: ColumnRef) -> bool:
         return self.col == other.col and self.perform_validation == other.perform_validation
 
+    def _df(self) -> 'pxt.dataframe.DataFrame':
+        tbl = catalog.Catalog.get().tbls[self.col.tbl.id]
+        return tbl.select(self)
+
+    def show(self, *args, **kwargs) -> 'pxt.dataframe.DataFrameResultSet':
+        return self._df().show(*args, **kwargs)
+
+    def head(self, *args, **kwargs) -> 'pxt.dataframe.DataFrameResultSet':
+        return self._df().head(*args, **kwargs)
+
+    def tail(self, *args, **kwargs) -> 'pxt.dataframe.DataFrameResultSet':
+        return self._df().tail(*args, **kwargs)
+
+    def count(self) -> int:
+        return self._df().count()
+
     def __str__(self) -> str:
         if self.col.name is None:
             return f'<unnamed column {self.col.id}>'
@@ -133,11 +151,20 @@ class ColumnRef(Expr):
             return self.col.name
 
     def __repr__(self) -> str:
-        return f'ColumnRef({self.col!r})'
+        return self._descriptors().to_string()
 
     def _repr_html_(self) -> str:
+        return self._descriptors().to_html()
+
+    def _descriptors(self) -> DescriptionHelper:
         tbl = catalog.Catalog.get().tbls[self.col.tbl.id]
-        return tbl._description_html(cols=[self.col])._repr_html_()  # type: ignore[attr-defined]
+        helper = DescriptionHelper()
+        helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path!r})')
+        helper.append(tbl._col_descriptor([self.col.name]))
+        idxs = tbl._index_descriptor([self.col.name])
+        if len(idxs) > 0:
+            helper.append(idxs)
+        return helper
 
     def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
         return None if self.perform_validation else self.col.sa_col

pixeltable/exprs/comparison.py

@@ -49,7 +49,7 @@ class Comparison(Expr):
 
         self.id = self._create_id()
 
-    def __str__(self) -> str:
+    def __repr__(self) -> str:
         return f'{self._op1} {self.operator} {self._op2}'
 
     def _equals(self, other: Comparison) -> bool:

pixeltable/exprs/compound_predicate.py

@@ -30,7 +30,7 @@ class CompoundPredicate(Expr):
 
         self.id = self._create_id()
 
-    def __str__(self) -> str:
+    def __repr__(self) -> str:
         if self.operator == LogicalOperator.NOT:
             return f'~({self.components[0]})'
         return f' {self.operator} '.join([f'({e})' for e in self.components])

pixeltable/exprs/expr.py

@@ -190,6 +190,7 @@ class Expr(abc.ABC):
                 return new.copy()
         for i in range(len(self.components)):
             self.components[i] = self.components[i].substitute(spec)
+        self.id = self._create_id()
         return self
 
     @classmethod
@@ -216,12 +217,12 @@ class Expr(abc.ABC):
                 return result
             result = result.substitute({ref: ref.col.value_expr for ref in target_col_refs})
 
-    def is_bound_by(self, tbl: catalog.TableVersionPath) -> bool:
-        """Returns True if this expr can be evaluated in the context of tbl."""
+    def is_bound_by(self, tbls: list[catalog.TableVersionPath]) -> bool:
+        """Returns True if this expr can be evaluated in the context of tbls."""
         from .column_ref import ColumnRef
         col_refs = self.subexprs(ColumnRef)
         for col_ref in col_refs:
-            if not tbl.has_column(col_ref.col):
+            if not any(tbl.has_column(col_ref.col) for tbl in tbls):
                 return False
         return True
 
@@ -235,7 +236,7 @@ class Expr(abc.ABC):
             self.components[i] = self.components[i]._retarget(tbl_versions)
         return self
 
-    def __str__(self) -> str:
+    def __repr__(self) -> str:
         return f'<Expression of type {type(self)}>'
 
     def display_str(self, inline: bool = True) -> str:
@@ -450,7 +451,13 @@ class Expr(abc.ABC):
 
     def astype(self, new_type: Union[ts.ColumnType, type, _AnnotatedAlias]) -> 'exprs.TypeCast':
         from pixeltable.exprs import TypeCast
-        return TypeCast(self, ts.ColumnType.normalize_type(new_type))
+        # Interpret the type argument the same way we would if given in a schema
+        col_type = ts.ColumnType.normalize_type(new_type, nullable_default=True, allow_builtin_types=False)
+        if not self.col_type.nullable:
+            # This expression is non-nullable; we can prove that the output is non-nullable, regardless of
+            # whether new_type is given as nullable.
+            col_type = col_type.copy(nullable=False)
+        return TypeCast(self, col_type)
 
     def apply(self, fn: Callable, *, col_type: Union[ts.ColumnType, type, _AnnotatedAlias, None] = None) -> 'exprs.FunctionCall':
         if col_type is not None:

pixeltable/exprs/expr_set.py

@@ -60,6 +60,14 @@ class ExprSet(Generic[T]):
     def __le__(self, other: ExprSet[T]) -> bool:
         return other.issuperset(self)
 
+    def union(self, *others: Iterable[T]) -> ExprSet[T]:
+        result = ExprSet(self.exprs.values())
+        result.update(*others)
+        return result
+
+    def __or__(self, other: ExprSet[T]) -> ExprSet[T]:
+        return self.union(other)
+
     def difference(self, *others: Iterable[T]) -> ExprSet[T]:
         id_diff = set(self.exprs.keys()).difference(e.id for other_set in others for e in other_set)
         return ExprSet(self.exprs[id] for id in id_diff)