pixeltable 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

pixeltable/__init__.py

@@ -19,6 +19,7 @@ from .globals import (
     list_functions,
     list_tables,
     move,
+    publish_snapshot,
     tool,
     tools,
 )

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = '0.3.4'
-__version_tuple__ = (0, 3, 4)
+__version__ = '0.3.6'
+__version_tuple__ = (0, 3, 6)

pixeltable/catalog/catalog.py

@@ -58,7 +58,7 @@ class Catalog:
             if session.query(sql.func.count(schema.Dir.id)).scalar() > 0:
                 return
             # create a top-level directory, so that every schema object has a directory
-            dir_md = schema.DirMd(name='')
+            dir_md = schema.DirMd(name='', user=None, additional_md={})
             dir_record = schema.Dir(parent_id=None, md=dataclasses.asdict(dir_md))
             session.add(dir_record)
             session.flush()
@@ -142,6 +142,7 @@ class Catalog:
                 base_tbl_id = base_path.tbl_id()
                 is_snapshot = view_md is not None and view_md.is_snapshot
                 snapshot_only = is_snapshot and view_md.predicate is None and len(schema_version_md.columns) == 0
+                include_base_columns = view_md is None or view_md.include_base_columns
                 if snapshot_only:
                     # this is a pure snapshot, without a physical table backing it
                     view_path = base_path
@@ -158,7 +159,13 @@ class Catalog:
                     view_path = TableVersionPath(tbl_version, base=base_path)
 
                 tbl: Table = View(
-                    tbl_record.id, tbl_record.dir_id, tbl_md.name, view_path, base_tbl_id, snapshot_only=snapshot_only
+                    tbl_record.id,
+                    tbl_record.dir_id,
+                    tbl_md.name,
+                    view_path,
+                    base_tbl_id,
+                    snapshot_only=snapshot_only,
+                    include_base_columns=include_base_columns,
                 )
                 self.tbl_dependents[base_tbl_id].append(tbl)
 

pixeltable/catalog/column.py

@@ -72,7 +72,7 @@ class Column:
           col_type is None
         - when loaded from md store: ``computed_with`` is set and col_type is set
 
-        ``stored`` (only valid for computed image columns):
+        ``stored`` (only valid for computed columns):
         - if True: the column is present in the stored table
         - if False: the column is not present in the stored table and recomputed during a query
         - if None: the system chooses for you (at present, this is always False, but this may change in the future)

pixeltable/catalog/dir.py

@@ -32,7 +32,7 @@ class Dir(SchemaObject):
     def _move(self, new_name: str, new_dir_id: UUID) -> None:
         super()._move(new_name, new_dir_id)
         with Env.get().engine.begin() as conn:
-            dir_md = schema.DirMd(name=new_name)
+            dir_md = schema.DirMd(name=new_name, user=None, additional_md={})
             conn.execute(
                 sql.update(schema.Dir.__table__)
                 .values({schema.Dir.parent_id: self._dir_id, schema.Dir.md: dataclasses.asdict(dir_md)})

pixeltable/catalog/table.py

@@ -569,7 +569,7 @@ class Table(SchemaObject):
 
         Args:
             kwargs: Exactly one keyword argument of the form `col_name=expression`.
-            stored: Whether the column is materialized and stored or computed on demand. Only valid for image columns.
+            stored: Whether the column is materialized and stored or computed on demand.
             print_stats: If `True`, print execution metrics during evaluation.
             on_error: Determines the behavior if an error occurs while evaluating the column expression for at least one
                 row.
@@ -682,6 +682,7 @@ class Table(SchemaObject):
             elif isinstance(spec, exprs.Expr):
                 # create copy so we can modify it
                 value_expr = spec.copy()
+                value_expr.bind_rel_paths()
             elif isinstance(spec, dict):
                 cls._validate_column_spec(name, spec)
                 if 'type' in spec:
@@ -692,6 +693,7 @@ class Table(SchemaObject):
                 if value_expr is not None and isinstance(value_expr, exprs.Expr):
                     # create copy so we can modify it
                     value_expr = value_expr.copy()
+                    value_expr.bind_rel_paths()
                 stored = spec.get('stored', True)
                 primary_key = spec.get('primary_key')
                 media_validation_str = spec.get('media_validation')

pixeltable/catalog/table_version.py

@@ -59,6 +59,7 @@ class TableVersion:
     schema_version: int
     view_md: Optional[schema.ViewMd]
     is_snapshot: bool
+    include_base_columns: bool
     effective_version: Optional[int]
     path: Optional[pxt.catalog.TableVersionPath]
     base: Optional[TableVersion]
@@ -115,6 +116,7 @@ class TableVersion:
         self.view_md = tbl_md.view_md  # save this as-is, it's needed for _create_md()
         is_view = tbl_md.view_md is not None
         self.is_snapshot = (is_view and tbl_md.view_md.is_snapshot) or bool(is_snapshot)
+        self.include_base_columns = not is_view or tbl_md.view_md.include_base_columns
         self.media_validation = MediaValidation[schema_version_md.media_validation.upper()]
         # a mutable TableVersion doesn't have a static version
         self.effective_version = self.version if self.is_snapshot else None
@@ -228,6 +230,7 @@ class TableVersion:
         column_md = cls._create_column_md(cols)
         table_md = schema.TableMd(
             name=name,
+            user=None,
             current_version=0,
             current_schema_version=0,
             next_col_id=len(cols),
@@ -237,6 +240,7 @@ class TableVersion:
             index_md={},
             external_stores=[],
             view_md=view_md,
+            additional_md={},
         )
         # create a schema.Table here, we need it to call our c'tor;
         # don't add it to the session yet, we might add index metadata
@@ -244,7 +248,7 @@ class TableVersion:
         tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(table_md))
 
         # create schema.TableVersion
-        table_version_md = schema.TableVersionMd(created_at=timestamp, version=0, schema_version=0)
+        table_version_md = schema.TableVersionMd(created_at=timestamp, version=0, schema_version=0, additional_md={})
         tbl_version_record = schema.TableVersion(
             tbl_id=tbl_record.id, version=0, md=dataclasses.asdict(table_version_md)
         )
@@ -266,6 +270,7 @@ class TableVersion:
             num_retained_versions=num_retained_versions,
             comment=comment,
             media_validation=media_validation.name.lower(),
+            additional_md={},
         )
         schema_version_record = schema.TableSchemaVersion(
             tbl_id=tbl_record.id, schema_version=0, md=dataclasses.asdict(schema_version_md)
@@ -1342,6 +1347,7 @@ class TableVersion:
     def _create_tbl_md(self) -> schema.TableMd:
         return schema.TableMd(
             name=self.name,
+            user=None,
             current_version=self.version,
             current_schema_version=self.schema_version,
             next_col_id=self.next_col_id,
@@ -1351,10 +1357,13 @@ class TableVersion:
             index_md=self.idx_md,
             external_stores=self._create_stores_md(self.external_stores.values()),
             view_md=self.view_md,
+            additional_md={},
         )
 
     def _create_version_md(self, timestamp: float) -> schema.TableVersionMd:
-        return schema.TableVersionMd(created_at=timestamp, version=self.version, schema_version=self.schema_version)
+        return schema.TableVersionMd(
+            created_at=timestamp, version=self.version, schema_version=self.schema_version, additional_md={}
+        )
 
     def _create_schema_version_md(self, preceding_schema_version: int) -> schema.TableSchemaVersionMd:
         column_md: dict[int, schema.SchemaColumn] = {}
@@ -1372,6 +1381,7 @@ class TableVersion:
             num_retained_versions=self.num_retained_versions,
             comment=self.comment,
             media_validation=self.media_validation.name.lower(),
+            additional_md={},
         )
 
     def as_dict(self) -> dict:

pixeltable/catalog/table_version_path.py

@@ -86,7 +86,7 @@ class TableVersionPath:
         from pixeltable.exprs import ColumnRef
 
         if col_name not in self.tbl_version.cols_by_name:
-            if self.base is None:
+            if self.base is None or not self.tbl_version.include_base_columns:
                 raise AttributeError(f'Column {col_name} unknown')
             return self.base.get_column_ref(col_name)
         col = self.tbl_version.cols_by_name[col_name]
@@ -95,7 +95,7 @@ class TableVersionPath:
     def columns(self) -> list[Column]:
         """Return all user columns visible in this tbl version path, including columns from bases"""
         result = list(self.tbl_version.cols_by_name.values())
-        if self.base is not None:
+        if self.base is not None and self.tbl_version.include_base_columns:
             base_cols = self.base.columns()
             # we only include base columns that don't conflict with one of our column names
             result.extend(c for c in base_cols if c.name not in self.tbl_version.cols_by_name)

pixeltable/catalog/view.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, Iterable, Literal, Optional
+from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional
 from uuid import UUID
 
 import sqlalchemy.orm as orm
@@ -37,7 +37,14 @@ class View(Table):
     """
 
     def __init__(
-        self, id: UUID, dir_id: UUID, name: str, tbl_version_path: TableVersionPath, base_id: UUID, snapshot_only: bool
+        self,
+        id: UUID,
+        dir_id: UUID,
+        name: str,
+        tbl_version_path: TableVersionPath,
+        base_id: UUID,
+        snapshot_only: bool,
+        include_base_columns: bool,
     ):
         super().__init__(id, dir_id, name, tbl_version_path)
         assert base_id in catalog.Catalog.get().tbl_dependents
@@ -48,12 +55,28 @@ class View(Table):
     def _display_name(cls) -> str:
         return 'view'
 
+    @classmethod
+    def select_list_to_additional_columns(cls, select_list: list[tuple[exprs.Expr, Optional[str]]]) -> dict[str, dict]:
+        """Returns a list of columns in the same format as the additional_columns parameter of View.create.
+        The source is the list of expressions from a select() statement on a DataFrame.
+        If the column is a ColumnRef, to a base table column, it is marked to not be stored.sy
+        """
+        from pixeltable.dataframe import DataFrame
+
+        r: dict[str, dict] = {}
+        exps, names = DataFrame._normalize_select_list([], select_list)
+        for expr, name in zip(exps, names):
+            stored = not isinstance(expr, exprs.ColumnRef)
+            r[name] = {'value': expr, 'stored': stored}
+        return r
+
     @classmethod
     def _create(
         cls,
         dir_id: UUID,
         name: str,
         base: TableVersionPath,
+        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]],
         additional_columns: dict[str, Any],
         predicate: Optional['pxt.exprs.Expr'],
         is_snapshot: bool,
@@ -63,7 +86,15 @@ class View(Table):
         iterator_cls: Optional[type[ComponentIterator]],
         iterator_args: Optional[dict],
     ) -> View:
-        columns = cls._create_columns(additional_columns)
+        # Convert select_list to more additional_columns if present
+        include_base_columns: bool = select_list is None
+        select_list_columns: List[Column] = []
+        if not include_base_columns:
+            r = cls.select_list_to_additional_columns(select_list)
+            select_list_columns = cls._create_columns(r)
+
+        columns_from_additional_columns = cls._create_columns(additional_columns)
+        columns = select_list_columns + columns_from_additional_columns
         cls._verify_schema(columns)
 
         # verify that filter can be evaluated in the context of the base
@@ -88,31 +119,34 @@ class View(Table):
 
             # validate iterator_args
             py_signature = inspect.signature(iterator_cls.__init__)
+
+            # make sure iterator_args can be used to instantiate iterator_cls
+            bound_args: dict[str, Any]
             try:
-                # make sure iterator_args can be used to instantiate iterator_cls
                 bound_args = py_signature.bind(None, **iterator_args).arguments  # None: arg for self
-                # we ignore 'self'
-                first_param_name = next(iter(py_signature.parameters))  # can't guarantee it's actually 'self'
-                del bound_args[first_param_name]
-
-                # construct Signature and type-check bound_args
-                params = [
-                    func.Parameter(param_name, param_type, kind=inspect.Parameter.POSITIONAL_OR_KEYWORD)
-                    for param_name, param_type in iterator_cls.input_schema().items()
-                ]
-                sig = func.Signature(ts.InvalidType(), params)
-                from pixeltable.exprs import FunctionCall
-
-                FunctionCall.normalize_args(iterator_cls.__name__, sig, bound_args)
             except TypeError as e:
-                raise excs.Error(f'Cannot instantiate iterator with given arguments: {e}')
+                raise excs.Error(f'Invalid iterator arguments: {e}')
+            # we ignore 'self'
+            first_param_name = next(iter(py_signature.parameters))  # can't guarantee it's actually 'self'
+            del bound_args[first_param_name]
+
+            # construct Signature and type-check bound_args
+            params = [
+                func.Parameter(param_name, param_type, kind=inspect.Parameter.POSITIONAL_OR_KEYWORD)
+                for param_name, param_type in iterator_cls.input_schema().items()
+            ]
+            sig = func.Signature(ts.InvalidType(), params)
+
+            expr_args = {k: exprs.Expr.from_object(v) for k, v in bound_args.items()}
+            sig.validate_args(expr_args, context=f'in iterator {iterator_cls.__name__!r}')
+            literal_args = {k: v.val if isinstance(v, exprs.Literal) else v for k, v in expr_args.items()}
 
             # prepend pos and output_schema columns to cols:
             # a component view exposes the pos column of its rowid;
             # we create that column here, so it gets assigned a column id;
             # stored=False: it is not stored separately (it's already stored as part of the rowid)
             iterator_cols = [Column(_POS_COLUMN_NAME, ts.IntType(), stored=False)]
-            output_dict, unstored_cols = iterator_cls.output_schema(**bound_args)
+            output_dict, unstored_cols = iterator_cls.output_schema(**literal_args)
             iterator_cols.extend(
                 [
                     Column(col_name, col_type, stored=col_name not in unstored_cols)
@@ -153,6 +187,7 @@ class View(Table):
 
             view_md = md_schema.ViewMd(
                 is_snapshot=is_snapshot,
+                include_base_columns=include_base_columns,
                 predicate=predicate.as_dict() if predicate is not None else None,
                 base_versions=base_versions,
                 iterator_class_fqn=iterator_class_fqn,
@@ -172,7 +207,15 @@ class View(Table):
             )
             if tbl_version is None:
                 # this is purely a snapshot: we use the base's tbl version path
-                view = cls(id, dir_id, name, base_version_path, base.tbl_id(), snapshot_only=True)
+                view = cls(
+                    id,
+                    dir_id,
+                    name,
+                    base_version_path,
+                    base.tbl_id(),
+                    snapshot_only=True,
+                    include_base_columns=include_base_columns,
+                )
                 _logger.info(f'created snapshot {name}')
             else:
                 view = cls(
@@ -182,6 +225,7 @@ class View(Table):
                     TableVersionPath(tbl_version, base=base_version_path),
                     base.tbl_id(),
                     snapshot_only=False,
+                    include_base_columns=include_base_columns,
                 )
                 _logger.info(f'Created view `{name}`, id={tbl_version.id}')
 

pixeltable/dataframe.py

@@ -293,7 +293,7 @@ class DataFrame:
             group_by_clause = self.group_by_clause
 
         for item in self._select_list_exprs:
-            item.bind_rel_paths(None)
+            item.bind_rel_paths()
 
         return plan.Planner.create_query_plan(
             self._from_clause,
@@ -951,7 +951,7 @@ class DataFrame:
 
             >>> df = person.where(t.year == 2014).update({'age': 30})
         """
-        self._validate_mutable('update')
+        self._validate_mutable('update', False)
         return self._first_tbl.tbl_version.update(value_spec, where=self.where_clause, cascade=cascade)
 
     def delete(self) -> UpdateStatus:
@@ -971,18 +971,23 @@ class DataFrame:
 
             >>> df = person.where(t.age < 18).delete()
         """
-        self._validate_mutable('delete')
+        self._validate_mutable('delete', False)
         if not self._first_tbl.is_insertable():
             raise excs.Error(f'Cannot delete from view')
         return self._first_tbl.tbl_version.delete(where=self.where_clause)
 
-    def _validate_mutable(self, op_name: str) -> None:
-        """Tests whether this DataFrame can be mutated (such as by an update operation)."""
+    def _validate_mutable(self, op_name: str, allow_select: bool) -> None:
+        """Tests whether this DataFrame can be mutated (such as by an update operation).
+
+        Args:
+            op_name: The name of the operation for which the test is being performed.
+            allow_select: If True, allow a select() specification in the Dataframe.
+        """
         if self.group_by_clause is not None or self.grouping_tbl is not None:
             raise excs.Error(f'Cannot use `{op_name}` after `group_by`')
         if self.order_by_clause is not None:
             raise excs.Error(f'Cannot use `{op_name}` after `order_by`')
-        if self.select_list is not None:
+        if self.select_list is not None and not allow_select:
             raise excs.Error(f'Cannot use `{op_name}` after `select`')
         if self.limit_val is not None:
             raise excs.Error(f'Cannot use `{op_name}` after `limit`')

pixeltable/env.py

@@ -76,6 +76,8 @@ class Env:
     _module_log_level: dict[str, int]  # module name -> log level
     _config_file: Optional[Path]
     _config: Optional[Config]
+    _file_cache_size_g: float
+    _pxt_api_key: Optional[str]
     _stdout_handler: logging.StreamHandler
     _initialized: bool
 
@@ -289,6 +291,7 @@ class Env:
                 f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {self._config_file},\n'
                 'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
             )
+        self._pxt_api_key = self._config.get_string_value('api_key')
 
         # Disable spurious warnings
         warnings.simplefilter('ignore', category=TqdmWarning)
@@ -459,6 +462,15 @@ class Env:
     def _upgrade_metadata(self) -> None:
         metadata.upgrade_md(self._sa_engine)
 
+    @property
+    def pxt_api_key(self) -> str:
+        if self._pxt_api_key is None:
+            raise excs.Error(
+                'No API key is configured. Set the PIXELTABLE_API_KEY environment variable, or add an entry to '
+                f'config.toml as described here:\nhttps://pixeltable.github.io/pixeltable/config/'
+            )
+        return self._pxt_api_key
+
     def get_client(self, name: str) -> Any:
         """
         Gets the client with the specified name, initializing it if necessary.

pixeltable/exec/expr_eval/evaluators.py

@@ -160,8 +160,10 @@ class FnCallEvaluator(Evaluator):
 
     def _create_batch_call_args(self, call_args: list[FnCallArgs]) -> FnCallArgs:
         """Roll call_args into a single batched FnCallArgs"""
-        batch_args: list[list[Optional[Any]]] = [[None] * len(call_args) for _ in range(len(self.fn_call.args))]
-        batch_kwargs: dict[str, list[Optional[Any]]] = {k: [None] * len(call_args) for k in self.fn_call.kwargs.keys()}
+        batch_args: list[list[Optional[Any]]] = [[None] * len(call_args) for _ in range(len(self.fn_call.arg_idxs))]
+        batch_kwargs: dict[str, list[Optional[Any]]] = {
+            k: [None] * len(call_args) for k in self.fn_call.kwarg_idxs.keys()
+        }
         assert isinstance(self.fn, func.CallableFunction)
         for i, item in enumerate(call_args):
             for j in range(len(item.args)):

pixeltable/exec/expr_eval/expr_eval_node.py

@@ -308,7 +308,10 @@ class ExprEvalNode(ExecNode):
                 if self.exc_event.is_set():
                     # we got an exception that we need to propagate through __iter__()
                     _logger.debug(f'Propagating exception {self.error}')
-                    raise self.error
+                    if isinstance(self.error, excs.ExprEvalError):
+                        raise self.error from self.error.exc
+                    else:
+                        raise self.error
                 if completed_aw in done:
                     self._log_state('completed_aw done')
                     completed_aw = None

pixeltable/exprs/comparison.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 import sqlalchemy as sql
 
 import pixeltable.exceptions as excs
-import pixeltable.index as index
 import pixeltable.type_system as ts
 
 from .column_ref import ColumnRef
@@ -16,12 +15,17 @@ from .literal import Literal
 from .row_builder import RowBuilder
 from .sql_element_cache import SqlElementCache
 
+if TYPE_CHECKING:
+    from pixeltable import index
+
 
 class Comparison(Expr):
     is_search_arg_comparison: bool
     operator: ComparisonOperator
 
     def __init__(self, operator: ComparisonOperator, op1: Expr, op2: Expr):
+        from pixeltable import index
+
         super().__init__(ts.BoolType())
         self.operator = operator
 
@@ -38,8 +42,6 @@ class Comparison(Expr):
             self.is_search_arg_comparison = False
             self.components = [op1, op2]
 
-        import pixeltable.index as index
-
         if (
             self.is_search_arg_comparison
             and self._op2.col_type.is_string_type()
@@ -71,6 +73,8 @@ class Comparison(Expr):
         return self.components[1]
 
     def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+        from pixeltable import index
+
         if str(self._op1.col_type.to_sa_type()) != str(self._op2.col_type.to_sa_type()):
             # Comparing columns of different SQL types (e.g., string vs. json); this can only be done in Python
             # TODO(aaron-siegel): We may be able to handle some cases in SQL by casting one side to the other's type

pixeltable/exprs/data_row.py

@@ -4,6 +4,7 @@ import datetime
 import io
 import urllib.parse
 import urllib.request
+from pathlib import Path
 from typing import Any, Optional
 
 import numpy as np
@@ -141,13 +142,13 @@ class DataRow:
         self.file_paths[slot_idx] = None
         self.file_urls[slot_idx] = None
 
-    def __getitem__(self, index: object) -> Any:
+    def __getitem__(self, index: int) -> Any:
         """Returns in-memory value, ie, what is needed for expr evaluation"""
         assert isinstance(index, int)
         if not self.has_val[index]:
-            # for debugging purposes
-            pass
-        assert self.has_val[index], index
+            # This is a sufficiently cheap and sensitive validation that it makes sense to keep the assertion around
+            # even if python is running with -O.
+            raise AssertionError(index)
 
         if self.file_urls[index] is not None and index in self.img_slot_idxs:
             # if we need to load this from a file, it should have been materialized locally
@@ -206,9 +207,10 @@ class DataRow:
                 # local file path
                 assert self.file_urls[idx] is None and self.file_paths[idx] is None
                 if len(parsed.scheme) <= 1:
-                    self.file_urls[idx] = urllib.parse.urljoin('file:', urllib.request.pathname2url(val))
-                    self.file_paths[idx] = val
-                else:
+                    path = str(Path(val).absolute())  # Ensure we're using an absolute pathname.
+                    self.file_urls[idx] = urllib.parse.urljoin('file:', urllib.request.pathname2url(path))
+                    self.file_paths[idx] = path
+                else:  # file:// URL
                     self.file_urls[idx] = val
                     # Wrap the path in a url2pathname() call to ensure proper handling on Windows.
                     self.file_paths[idx] = urllib.parse.unquote(urllib.request.url2pathname(parsed.path))

pixeltable/exprs/expr.py

@@ -583,7 +583,7 @@ class Expr(abc.ABC):
 
     def __bool__(self) -> bool:
         raise TypeError(
-            'Pixeltable expressions cannot be used in conjunction with Python boolean operators (and/or/not)'
+            f'Pixeltable expressions cannot be used in conjunction with Python boolean operators (and/or/not)\n{self!r}'
         )
 
     def __lt__(self, other: object) -> 'exprs.Comparison':
@@ -784,7 +784,7 @@ class Expr(abc.ABC):
             if (
                 len(params) >= 2
                 and second_param.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
-                and second_param.default == inspect.Parameter.empty
+                and second_param.default is inspect.Parameter.empty
             ):
                 raise excs.Error(f'Function `{fn.__name__}` has multiple required parameters.')
         except ValueError: