pixeltable 0.2.26__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/exprs/globals.py

@@ -2,10 +2,11 @@ from __future__ import annotations
 
 import datetime
 import enum
-from typing import Union
+import uuid
 
 # Python types corresponding to our literal types
-LiteralPythonTypes = Union[str, int, float, bool, datetime.datetime]
+LiteralPythonTypes = str | int | float | bool | datetime.datetime | datetime.date | uuid.UUID
+
 
 def print_slice(s: slice) -> str:
     start_str = f'{str(s.start) if s.start is not None else ""}'
@@ -35,7 +36,7 @@ class ComparisonOperator(enum.Enum):
             return '>'
         if self == self.GE:
             return '>='
-        assert False
+        raise AssertionError()
 
     def reverse(self) -> ComparisonOperator:
         if self == self.LT:
@@ -61,7 +62,7 @@ class LogicalOperator(enum.Enum):
             return '|'
         if self == self.NOT:
             return '~'
-        assert False
+        raise AssertionError()
 
 
 class ArithmeticOperator(enum.Enum):
@@ -85,4 +86,16 @@ class ArithmeticOperator(enum.Enum):
             return '%'
         if self == self.FLOORDIV:
             return '//'
-        assert False
+        raise AssertionError()
+
+
+class StringOperator(enum.Enum):
+    CONCAT = 0
+    REPEAT = 1
+
+    def __str__(self) -> str:
+        if self == self.CONCAT:
+            return '+'
+        if self == self.REPEAT:
+            return '*'
+        raise AssertionError()

pixeltable/exprs/in_predicate.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Iterable, Optional
+from typing import Any, Iterable
 
 import sqlalchemy as sql
 
@@ -16,17 +16,18 @@ from .sql_element_cache import SqlElementCache
 class InPredicate(Expr):
     """Predicate corresponding to the SQL IN operator."""
 
-    def __init__(self, lhs: Expr, value_set_literal: Optional[Iterable] = None, value_set_expr: Optional[Expr] = None):
+    def __init__(self, lhs: Expr, value_set_literal: Iterable | None = None, value_set_expr: Expr | None = None):
         assert (value_set_literal is None) != (value_set_expr is None)
         if not lhs.col_type.is_scalar_type():
             raise excs.Error(f'isin(): only supported for scalar types, not {lhs.col_type}')
         super().__init__(ts.BoolType())
 
-        self.value_list: Optional[list] = None  # only contains values of the correct type
+        self.value_list: list | None = None  # only contains values of the correct type
         if value_set_expr is not None:
             if not value_set_expr.col_type.is_json_type():
                 raise excs.Error(
-                    f'isin(): argument must have a JSON type, but {value_set_expr} has type {value_set_expr.col_type}')
+                    f'isin(): argument must have a JSON type, but {value_set_expr} has type {value_set_expr.col_type}'
+                )
             self.components = [lhs.copy(), value_set_expr.copy()]
         else:
             assert value_set_literal is not None
@@ -70,9 +71,9 @@ class InPredicate(Expr):
         return self.value_list == other.value_list
 
     def _id_attrs(self) -> list[tuple[str, Any]]:
-        return super()._id_attrs() + [('value_list', self.value_list)]
+        return [*super()._id_attrs(), ('value_list', self.value_list)]
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
         lhs_sql_exprs = sql_elements.get(self.components[0])
         if lhs_sql_exprs is None or self.value_list is None:
             return None
@@ -95,4 +96,3 @@ class InPredicate(Expr):
         assert 'value_list' in d
         assert len(components) <= 2
         return cls(components[0], d['value_list'], components[1] if len(components) == 2 else None)
-

pixeltable/exprs/inline_expr.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-import copy
-from typing import Any, Iterable, Optional
+from typing import Any, Iterable
 
 import numpy as np
 import sqlalchemy as sql
@@ -26,14 +25,14 @@ class InlineArray(Expr):
         for el in elements:
             if isinstance(el, Expr):
                 exprs.append(el)
-            elif isinstance(el, list) or isinstance(el, tuple):
-                exprs.append(InlineArray(el))
+            elif isinstance(el, (list, tuple)):
+                exprs.append(Expr.from_array(el))
             else:
                 exprs.append(Literal(el))
 
-        inferred_element_type: Optional[ts.ColumnType] = ts.InvalidType()
+        inferred_element_type: ts.ColumnType | None = ts.InvalidType()
         for i, expr in enumerate(exprs):
-            supertype = inferred_element_type.supertype(expr.col_type)
+            supertype = inferred_element_type.supertype(expr.col_type, for_inference=True)
             if supertype is None:
                 raise excs.Error(
                     f'Could not infer element type of array: element of type `{expr.col_type}` at index {i} '
@@ -45,10 +44,12 @@ class InlineArray(Expr):
             col_type = ts.ArrayType((len(exprs),), inferred_element_type)
         elif inferred_element_type.is_array_type():
             assert isinstance(inferred_element_type, ts.ArrayType)
-            col_type = ts.ArrayType(
-                (len(exprs), *inferred_element_type.shape),
-                ts.ColumnType.make_type(inferred_element_type.dtype)
-            )
+            dtype = inferred_element_type.dtype
+            shape = inferred_element_type.shape
+            if shape is not None and dtype is not None:
+                col_type = ts.ArrayType(shape=(len(exprs), *shape), dtype=dtype)
+            else:
+                col_type = ts.ArrayType(shape=None, dtype=dtype)
         else:
             raise excs.Error(f'Element type is not a valid dtype for an array: {inferred_element_type}')
 
@@ -63,7 +64,7 @@ class InlineArray(Expr):
     def _equals(self, _: InlineArray) -> bool:
         return True  # Always true if components match
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
         return None
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
@@ -83,6 +84,14 @@ class InlineArray(Expr):
             # loaded and their types are known.
             return InlineList(components)  # type: ignore[return-value]
 
+    def as_literal(self) -> Literal | None:
+        assert isinstance(self.col_type, ts.ArrayType)
+        if not all(isinstance(comp, Literal) for comp in self.components):
+            return None
+        return Literal(
+            np.array([c.as_literal().val for c in self.components], dtype=self.col_type.dtype), self.col_type
+        )
+
 
 class InlineList(Expr):
     """
@@ -90,16 +99,7 @@ class InlineList(Expr):
     """
 
     def __init__(self, elements: Iterable):
-        exprs = []
-        for el in elements:
-            if isinstance(el, Expr):
-                exprs.append(el)
-            elif isinstance(el, list) or isinstance(el, tuple):
-                exprs.append(InlineList(el))
-            elif isinstance(el, dict):
-                exprs.append(InlineDict(el))
-            else:
-                exprs.append(Literal(el))
+        exprs = [Expr.from_object(el) for el in elements]
 
         super().__init__(ts.JsonType())
         self.components.extend(exprs)
@@ -112,7 +112,7 @@ class InlineList(Expr):
     def _equals(self, _: InlineList) -> bool:
         return True  # Always true if components match
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
         return None
 
     def eval(self, data_row: DataRow, _: RowBuilder) -> None:
@@ -125,6 +125,11 @@ class InlineList(Expr):
     def _from_dict(cls, _: dict, components: list[Expr]) -> InlineList:
         return cls(components)
 
+    def as_literal(self) -> Literal | None:
+        if not all(isinstance(comp, Literal) for comp in self.components):
+            return None
+        return Literal([c.as_literal().val for c in self.components], self.col_type)
+
 
 class InlineDict(Expr):
     """
@@ -140,21 +145,14 @@ class InlineDict(Expr):
             if not isinstance(key, str):
                 raise excs.Error(f'Dictionary requires string keys; {key} has type {type(key)}')
             self.keys.append(key)
-            if isinstance(val, Expr):
-                exprs.append(val)
-            elif isinstance(val, dict):
-                exprs.append(InlineDict(val))
-            elif isinstance(val, list) or isinstance(val, tuple):
-                exprs.append(InlineList(val))
-            else:
-                exprs.append(Literal(val))
+            exprs.append(Expr.from_object(val))
 
         super().__init__(ts.JsonType())
         self.components.extend(exprs)
         self.id = self._create_id()
 
     def __repr__(self) -> str:
-        item_strs = list(f"'{key}': {str(expr)}" for key, expr in zip(self.keys, self.components))
+        item_strs = [f"'{key}': {expr}" for key, expr in zip(self.keys, self.components)]
         return '{' + ', '.join(item_strs) + '}'
 
     def _equals(self, other: InlineDict) -> bool:
@@ -162,17 +160,14 @@ class InlineDict(Expr):
         return self.keys == other.keys
 
     def _id_attrs(self) -> list[tuple[str, Any]]:
-        return super()._id_attrs() + [('keys', self.keys)]
+        return [*super()._id_attrs(), ('keys', self.keys)]
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
         return None
 
     def eval(self, data_row: DataRow, _: RowBuilder) -> None:
         assert len(self.keys) == len(self.components)
-        data_row[self.slot_idx] = {
-            key: data_row[expr.slot_idx]
-            for key, expr in zip(self.keys, self.components)
-        }
+        data_row[self.slot_idx] = {key: data_row[expr.slot_idx] for key, expr in zip(self.keys, self.components)}
 
     def to_kwargs(self) -> dict[str, Any]:
         """Deconstructs this expression into a dictionary by recursively unwrapping all Literals,
@@ -198,3 +193,8 @@ class InlineDict(Expr):
         assert len(d['keys']) == len(components)
         arg = dict(zip(d['keys'], components))
         return InlineDict(arg)
+
+    def as_literal(self) -> Literal | None:
+        if not all(isinstance(comp, Literal) for comp in self.components):
+            return None
+        return Literal(dict(zip(self.keys, (c.as_literal().val for c in self.components))), self.col_type)

pixeltable/exprs/is_null.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
-from typing import Optional
-
 import sqlalchemy as sql
 
 import pixeltable.type_system as ts
 
+from .column_ref import ColumnRef
 from .data_row import DataRow
 from .expr import Expr
 from .row_builder import RowBuilder
@@ -19,12 +18,17 @@ class IsNull(Expr):
         self.id = self._create_id()
 
     def __repr__(self) -> str:
-        return f'{str(self.components[0])} == None'
+        return f'{self.components[0]} == None'
 
     def _equals(self, other: IsNull) -> bool:
         return True
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
+        c = self.components[0]
+        if isinstance(c, ColumnRef) and c.col.stores_external_array():
+            # we also need to check CellMd.file_urls for null
+            e = sql.and_(c.col.sa_cellmd_col['file_urls'] == None, c.col.sa_col == None)
+            return e
         e = sql_elements.get(self.components[0])
         if e is None:
             return None

pixeltable/exprs/json_mapper.py

@@ -1,51 +1,143 @@
 from __future__ import annotations
 
-from typing import Optional
+from typing import TYPE_CHECKING
 
 import sqlalchemy as sql
 
 import pixeltable.type_system as ts
+
 from .data_row import DataRow
-from .expr import Expr, ExprScope, _GLOBAL_SCOPE
+from .expr import _GLOBAL_SCOPE, Expr, ExprScope
 from .row_builder import RowBuilder
 from .sql_element_cache import SqlElementCache
 
+if TYPE_CHECKING:
+    from .object_ref import ObjectRef
+
 
 class JsonMapper(Expr):
     """
     JsonMapper transforms the list output of a JsonPath by applying a target expr to every element of the list.
     The target expr would typically contain relative JsonPaths, which are bound to an ObjectRef, which in turn
     is populated by JsonMapper.eval(). The JsonMapper effectively creates a new scope for its target expr.
+
+    JsonMapper is executed in two phases:
+    - the first phase is handled by Expr subclass JsonMapperDispatch, which constructs one nested DataRow per source
+      list element and evaluates the target expr within that (the nested DataRows are stored as a NestedRowList in the
+      slot of JsonMapperDispatch)
+    - JsonMapper.eval() collects the slot values of the target expr into its result list
     """
-    def __init__(self, src_expr: Expr, target_expr: Expr):
+
+    target_expr_scope: ExprScope
+    parent_mapper: JsonMapper | None
+    target_expr_eval_ctx: RowBuilder.EvalCtx | None
+
+    def __init__(self, src_expr: Expr | None, target_expr: Expr | None):
         # TODO: type spec should be list[target_expr.col_type]
         super().__init__(ts.JsonType())
 
+        dispatch = JsonMapperDispatch(src_expr, target_expr)
+        self.components.append(dispatch)
+        self.id = self._create_id()
+
+    def __repr__(self) -> str:
+        return f'map({self._src_expr}, lambda R: {self._target_expr})'
+
+    @property
+    def _src_expr(self) -> Expr:
+        return self.components[0].src_expr
+
+    @property
+    def _target_expr(self) -> Expr:
+        return self.components[0].target_expr
+
+    def _equals(self, _: JsonMapper) -> bool:
+        return True
+
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
+        return None
+
+    def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        from ..exec.expr_eval.evaluators import NestedRowList
+
+        dispatch_slot_idx = self.components[0].slot_idx
+        nested_rows = data_row.vals[dispatch_slot_idx]
+        if nested_rows is None:
+            data_row[self.slot_idx] = None
+            return
+        assert isinstance(nested_rows, NestedRowList)
+        # TODO: get the materialized slot idx, instead of relying on the fact that the target_expr is always at the end
+        data_row[self.slot_idx] = [row.vals[-1] for row in nested_rows.rows]
+
+    def _as_dict(self) -> dict:
+        """
+        We only serialize src and target exprs, everything else is re-created at runtime.
+        """
+        return {'components': [self._src_expr.as_dict(), self._target_expr.as_dict()]}
+
+    @classmethod
+    def _from_dict(cls, d: dict, components: list[Expr]) -> JsonMapper:
+        assert len(components) == 2
+        src_expr, target_expr = components[0], components[1]
+        return cls(src_expr, target_expr)
+
+
+class JsonMapperDispatch(Expr):
+    """
+    An operational Expr (ie, it doesn't represent any syntactic element) that is used by JsonMapper to materialize
+    its input DataRows. It has the same dependencies as the originating JsonMapper.
+
+    - The execution (= row dispatch) is handled by an expr_eval.Evaluator (JsonMapperDispatcher).
+    - It stores a NestedRowList instance in its slot.
+    """
+
+    target_expr_scope: ExprScope
+    parent_mapper: JsonMapperDispatch | None
+    target_expr_eval_ctx: RowBuilder.EvalCtx | None
+
+    def __init__(self, src_expr: Expr, target_expr: Expr):
+        super().__init__(ts.InvalidType())
+
         # we're creating a new scope, but we don't know yet whether this is nested within another JsonMapper;
         # this gets resolved in bind_rel_paths(); for now we assume we're in the global scope
         self.target_expr_scope = ExprScope(_GLOBAL_SCOPE)
 
         from .object_ref import ObjectRef
-        scope_anchor = ObjectRef(self.target_expr_scope, self)
-        self.components = [src_expr, target_expr, scope_anchor]
-        self.parent_mapper: Optional[JsonMapper] = None
-        self.target_expr_eval_ctx: Optional[RowBuilder.EvalCtx] = None
+
+        self.components = [src_expr, target_expr]
+        self.parent_mapper = None
+        self.target_expr_eval_ctx = None
+
+        # Intentionally create the id now, before adding the scope anchor; this ensures that JsonMapperDispatch
+        # instances will be recognized as equal so long as they have the same src_expr and target_expr.
+        # TODO: Might this cause problems after certain substitutions?
         self.id = self._create_id()
 
-    def bind_rel_paths(self, mapper: Optional[JsonMapper] = None) -> None:
-        self._src_expr.bind_rel_paths(mapper)
-        self._target_expr.bind_rel_paths(self)
+        scope_anchor = ObjectRef(self.target_expr_scope, self)
+        self.components.append(scope_anchor)
+
+    def _bind_rel_paths(self, mapper: JsonMapperDispatch | None = None) -> None:
+        self.src_expr._bind_rel_paths(mapper)
+        self.target_expr._bind_rel_paths(self)
         self.parent_mapper = mapper
         parent_scope = _GLOBAL_SCOPE if mapper is None else mapper.target_expr_scope
         self.target_expr_scope.parent = parent_scope
 
+    def equals(self, other: Expr) -> bool:
+        """
+        We override equals() because we need to avoid comparing our scope anchor.
+        """
+        if type(self) is not type(other):
+            return False
+        return self.src_expr.equals(other.src_expr) and self.target_expr.equals(other.target_expr)
+
     def scope(self) -> ExprScope:
         # need to ignore target_expr
-        return self._src_expr.scope()
+        return self.src_expr.scope()
 
     def dependencies(self) -> list[Expr]:
-        result = [self._src_expr]
-        result.extend(self._target_dependencies(self._target_expr))
+        result = [self.src_expr]
+        result.extend(self._target_dependencies(self.target_expr))
         return result
 
     def _target_dependencies(self, e: Expr) -> list[Expr]:
@@ -61,61 +153,35 @@ class JsonMapper(Expr):
             result.extend(self._target_dependencies(c))
         return result
 
-    def equals(self, other: Expr) -> bool:
-        """
-        We override equals() because we need to avoid comparing our scope anchor.
-        """
-        if type(self) != type(other):
-            return False
-        return self._src_expr.equals(other._src_expr) and self._target_expr.equals(other._target_expr)
-
-    def __repr__(self) -> str:
-        return f'{str(self._src_expr)} >> {str(self._target_expr)}'
-
     @property
-    def _src_expr(self) -> Expr:
+    def src_expr(self) -> Expr:
         return self.components[0]
 
     @property
-    def _target_expr(self) -> Expr:
+    def target_expr(self) -> Expr:
         return self.components[1]
 
     @property
-    def scope_anchor(self) -> Expr:
-        return self.components[2]
+    def scope_anchor(self) -> 'ObjectRef':
+        from .object_ref import ObjectRef
 
-    def _equals(self, _: JsonMapper) -> bool:
-        return True
+        result = self.components[2]
+        assert isinstance(result, ObjectRef)
+        return result
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
-        return None
+    def __repr__(self) -> str:
+        return 'JsonMapperDispatch()'
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
-        # this will be called, but the value has already been materialized elsewhere
-        src = data_row[self._src_expr.slot_idx]
-        if not isinstance(src, list):
-            # invalid/non-list src path
-            data_row[self.slot_idx] = None
-            return
-
-        result = [None] * len(src)
-        if self.target_expr_eval_ctx is None:
-            self.target_expr_eval_ctx = row_builder.create_eval_ctx([self._target_expr])
-        for i, val in enumerate(src):
-            data_row[self.scope_anchor.slot_idx] = val
-            # stored target_expr
-            row_builder.eval(data_row, self.target_expr_eval_ctx)
-            result[i] = data_row[self._target_expr.slot_idx]
-        data_row[self.slot_idx] = result
+        # eval is handled by JsonMapperDispatcher
+        raise AssertionError('this should never be called')
 
     def _as_dict(self) -> dict:
         """
-        We need to avoid serializing component[2], which is an ObjectRef.
+        JsonMapperDispatch instances are only created by the JsonMapper c'tor and never need to be serialized.
         """
-        return {'components': [c.as_dict() for c in self.components[0:2]]}
+        raise AssertionError('this should never be called')
 
     @classmethod
-    def _from_dict(cls, d: dict, components: list[Expr]) -> JsonMapper:
-        assert len(components) == 2
-        return cls(components[0], components[1])
-
+    def _from_dict(cls, d: dict, components: list[Expr]) -> JsonMapperDispatch:
+        raise AssertionError('this should never be called')