pixeltable 0.3.5__tar.gz → 0.3.6__tar.gz

{pixeltable-0.3.5 → pixeltable-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pixeltable
-Version: 0.3.5
+Version: 0.3.6
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
 License: Apache-2.0
 Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = '0.3.5'
-__version_tuple__ = (0, 3, 5)
+__version__ = '0.3.6'
+__version_tuple__ = (0, 3, 6)

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/catalog/table.py

@@ -682,6 +682,7 @@ class Table(SchemaObject):
             elif isinstance(spec, exprs.Expr):
                 # create copy so we can modify it
                 value_expr = spec.copy()
+                value_expr.bind_rel_paths()
             elif isinstance(spec, dict):
                 cls._validate_column_spec(name, spec)
                 if 'type' in spec:
@@ -692,6 +693,7 @@ class Table(SchemaObject):
                 if value_expr is not None and isinstance(value_expr, exprs.Expr):
                     # create copy so we can modify it
                     value_expr = value_expr.copy()
+                    value_expr.bind_rel_paths()
                 stored = spec.get('stored', True)
                 primary_key = spec.get('primary_key')
                 media_validation_str = spec.get('media_validation')

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/dataframe.py

@@ -293,7 +293,7 @@ class DataFrame:
             group_by_clause = self.group_by_clause
 
         for item in self._select_list_exprs:
-            item.bind_rel_paths(None)
+            item.bind_rel_paths()
 
         return plan.Planner.create_query_plan(
             self._from_clause,

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/exprs/data_row.py

@@ -142,13 +142,13 @@ class DataRow:
         self.file_paths[slot_idx] = None
         self.file_urls[slot_idx] = None
 
-    def __getitem__(self, index: object) -> Any:
+    def __getitem__(self, index: int) -> Any:
         """Returns in-memory value, ie, what is needed for expr evaluation"""
         assert isinstance(index, int)
         if not self.has_val[index]:
-            # for debugging purposes
-            pass
-        assert self.has_val[index], index
+            # This is a sufficiently cheap and sensitive validation that it makes sense to keep the assertion around
+            # even if python is running with -O.
+            raise AssertionError(index)
 
         if self.file_urls[index] is not None and index in self.img_slot_idxs:
             # if we need to load this from a file, it should have been materialized locally

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/exprs/json_mapper.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 import sqlalchemy as sql
 
@@ -11,6 +11,9 @@ from .expr import _GLOBAL_SCOPE, Expr, ExprScope
 from .row_builder import RowBuilder
 from .sql_element_cache import SqlElementCache
 
+if TYPE_CHECKING:
+    from .object_ref import ObjectRef
+
 
 class JsonMapper(Expr):
     """
@@ -19,6 +22,10 @@ class JsonMapper(Expr):
     is populated by JsonMapper.eval(). The JsonMapper effectively creates a new scope for its target expr.
     """
 
+    target_expr_scope: ExprScope
+    parent_mapper: Optional[JsonMapper]
+    target_expr_eval_ctx: Optional[RowBuilder.EvalCtx]
+
     def __init__(self, src_expr: Expr, target_expr: Expr):
         # TODO: type spec should be list[target_expr.col_type]
         super().__init__(ts.JsonType())
@@ -29,12 +36,18 @@ class JsonMapper(Expr):
 
         from .object_ref import ObjectRef
 
-        scope_anchor = ObjectRef(self.target_expr_scope, self)
-        self.components = [src_expr, target_expr, scope_anchor]
-        self.parent_mapper: Optional[JsonMapper] = None
-        self.target_expr_eval_ctx: Optional[RowBuilder.EvalCtx] = None
+        self.components = [src_expr, target_expr]
+        self.parent_mapper = None
+        self.target_expr_eval_ctx = None
+
+        # Intentionally create the id now, before adding the scope anchor; this ensures that JsonMappers will
+        # be recognized as equal so long as they have the same src_expr and target_expr.
+        # TODO: Might this cause problems after certain substitutions?
         self.id = self._create_id()
 
+        scope_anchor = ObjectRef(self.target_expr_scope, self)
+        self.components.append(scope_anchor)
+
     def bind_rel_paths(self, mapper: Optional[JsonMapper] = None) -> None:
         self._src_expr.bind_rel_paths(mapper)
         self._target_expr.bind_rel_paths(self)
@@ -84,8 +97,12 @@ class JsonMapper(Expr):
         return self.components[1]
 
     @property
-    def scope_anchor(self) -> Expr:
-        return self.components[2]
+    def scope_anchor(self) -> 'ObjectRef':
+        from .object_ref import ObjectRef
+
+        result = self.components[2]
+        assert isinstance(result, ObjectRef)
+        return result
 
     def _equals(self, _: JsonMapper) -> bool:
         return True
@@ -107,7 +124,7 @@ class JsonMapper(Expr):
         for i, val in enumerate(src):
             data_row[self.scope_anchor.slot_idx] = val
             # stored target_expr
-            row_builder.eval(data_row, self.target_expr_eval_ctx)
+            row_builder.eval(data_row, self.target_expr_eval_ctx, force_eval=self._target_expr.scope())
             result[i] = data_row[self._target_expr.slot_idx]
         data_row[self.slot_idx] = result
 

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/exprs/json_path.py

@@ -43,11 +43,11 @@ class JsonPath(Expr):
         self.id = self._create_id()
 
     def __repr__(self) -> str:
-        # else "R": the anchor is RELATIVE_PATH_ROOT
-        return (
-            f'{str(self._anchor) if self._anchor is not None else "R"}'
-            f'{"." if isinstance(self.path_elements[0], str) else ""}{self._json_path()}'
-        )
+        # else 'R': the anchor is RELATIVE_PATH_ROOT
+        anchor_str = str(self._anchor) if self._anchor is not None else 'R'
+        if len(self.path_elements) == 0:
+            return anchor_str
+        return f'{anchor_str}{"." if isinstance(self.path_elements[0], str) else ""}{self._json_path()}'
 
     def _as_dict(self) -> dict:
         path_elements = [[el.start, el.stop, el.step] if isinstance(el, slice) else el for el in self.path_elements]
@@ -158,6 +158,7 @@ class JsonPath(Expr):
         return ''.join(result)
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        assert self._anchor is not None, self
         val = data_row[self._anchor.slot_idx]
         if self.compiled_path is not None:
             val = self.compiled_path.search(val)

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/exprs/object_ref.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Optional
+from typing import Any, Optional
 
 import sqlalchemy as sql
 
@@ -26,14 +26,22 @@ class ObjectRef(Expr):
         self.owner = owner
         self.id = self._create_id()
 
+    def _id_attrs(self) -> list[tuple[str, Any]]:
+        # We have no components, so we can't rely on the default behavior here (otherwise, all ObjectRef
+        # instances will be conflated into a single slot).
+        return [('addr', id(self))]
+
+    def substitute(self, subs: dict[Expr, Expr]) -> Expr:
+        # Just return self; we need to avoid creating a new id after doing the substitution, because otherwise
+        # we'll wind up in a situation where the scope_anchor of the enclosing JsonMapper is different from the
+        # nested ObjectRefs inside its target_expr (and therefore occupies a different slot_idx).
+        return self
+
     def scope(self) -> ExprScope:
         return self._scope
 
-    def __str__(self) -> str:
-        assert False
-
     def _equals(self, other: ObjectRef) -> bool:
-        return self.owner is other.owner
+        return self.id == other.id
 
     def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
         return None
@@ -41,3 +49,6 @@ class ObjectRef(Expr):
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
         # this will be called, but the value has already been materialized elsewhere
         pass
+
+    def __repr__(self) -> str:
+        return f'ObjectRef({self.owner}, {self.id}, {self.owner.id})'

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/exprs/row_builder.py

@@ -17,7 +17,7 @@ from pixeltable.env import Env
 from pixeltable.utils.media_store import MediaStore
 
 from .data_row import DataRow
-from .expr import Expr
+from .expr import Expr, ExprScope
 from .expr_set import ExprSet
 
 
@@ -299,6 +299,7 @@ class RowBuilder:
                 # this is input and therefore doesn't depend on other exprs
                 continue
             for d in expr.dependencies():
+                assert d.slot_idx is not None, f'{expr}, {d}'
                 if d.slot_idx in excluded_slot_idxs:
                     continue
                 dependencies[expr.slot_idx].add(d.slot_idx)
@@ -376,7 +377,12 @@ class RowBuilder:
             data_row.set_exc(slot_idx, exc)
 
     def eval(
-        self, data_row: DataRow, ctx: EvalCtx, profile: Optional[ExecProfile] = None, ignore_errors: bool = False
+        self,
+        data_row: DataRow,
+        ctx: EvalCtx,
+        profile: Optional[ExecProfile] = None,
+        ignore_errors: bool = False,
+        force_eval: Optional[ExprScope] = None,
     ) -> None:
         """
         Populates the slots in data_row given in ctx.
@@ -384,10 +390,11 @@ class RowBuilder:
         and omits any of that expr's dependents's eval().
         profile: if present, populated with execution time of each expr.eval() call; indexed by expr.slot_idx
         ignore_errors: if False, raises ExprEvalError if any expr.eval() raises an exception
+        force_eval: forces exprs in the specified scope to be reevaluated, even if they already have a value
         """
         for expr in ctx.exprs:
             assert expr.slot_idx >= 0
-            if data_row.has_val[expr.slot_idx] or data_row.has_exc(expr.slot_idx):
+            if expr.scope() != force_eval and (data_row.has_val[expr.slot_idx] or data_row.has_exc(expr.slot_idx)):
                 continue
             try:
                 start_time = time.perf_counter()

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/func/tools.py

@@ -48,22 +48,27 @@ class Tool(pydantic.BaseModel):
             'additionalProperties': False,  # TODO Handle kwargs?
         }
 
-    # `tool_calls` must be in standardized tool invocation format:
-    # {tool_name: {'args': {name1: value1, name2: value2, ...}}, ...}
-    def invoke(self, tool_calls: 'exprs.Expr') -> 'exprs.FunctionCall':
-        kwargs = {param.name: self.__extract_tool_arg(param, tool_calls) for param in self.parameters.values()}
-        return self.fn(**kwargs)
+    # The output of `tool_calls` must be a dict in standardized tool invocation format:
+    # {tool_name: [{'args': {name1: value1, name2: value2, ...}}, ...], ...}
+    def invoke(self, tool_calls: 'exprs.Expr') -> 'exprs.Expr':
+        from pixeltable import exprs
 
-    def __extract_tool_arg(self, param: Parameter, tool_calls: 'exprs.Expr') -> 'exprs.Expr':
         func_name = self.name or self.fn.name
+        return exprs.JsonMapper(tool_calls[func_name]['*'], self.__invoke_kwargs(exprs.RELATIVE_PATH_ROOT.args))
+
+    def __invoke_kwargs(self, kwargs: 'exprs.Expr') -> 'exprs.FunctionCall':
+        kwargs = {param.name: self.__extract_tool_arg(param, kwargs) for param in self.parameters.values()}
+        return self.fn(**kwargs)
+
+    def __extract_tool_arg(self, param: Parameter, kwargs: 'exprs.Expr') -> 'exprs.FunctionCall':
         if param.col_type.is_string_type():
-            return _extract_str_tool_arg(tool_calls, func_name=func_name, param_name=param.name)
+            return _extract_str_tool_arg(kwargs, param_name=param.name)
         if param.col_type.is_int_type():
-            return _extract_int_tool_arg(tool_calls, func_name=func_name, param_name=param.name)
+            return _extract_int_tool_arg(kwargs, param_name=param.name)
         if param.col_type.is_float_type():
-            return _extract_float_tool_arg(tool_calls, func_name=func_name, param_name=param.name)
+            return _extract_float_tool_arg(kwargs, param_name=param.name)
         if param.col_type.is_bool_type():
-            return _extract_bool_tool_arg(tool_calls, func_name=func_name, param_name=param.name)
+            return _extract_bool_tool_arg(kwargs, param_name=param.name)
         assert False
 
 
@@ -113,34 +118,29 @@ class Tools(pydantic.BaseModel):
 
 
 @udf
-def _extract_str_tool_arg(tool_calls: dict[str, Any], func_name: str, param_name: str) -> Optional[str]:
-    return _extract_arg(str, tool_calls, func_name, param_name)
+def _extract_str_tool_arg(kwargs: dict[str, Any], param_name: str) -> Optional[str]:
+    return _extract_arg(str, kwargs, param_name)
 
 
 @udf
-def _extract_int_tool_arg(tool_calls: dict[str, Any], func_name: str, param_name: str) -> Optional[int]:
-    return _extract_arg(int, tool_calls, func_name, param_name)
+def _extract_int_tool_arg(kwargs: dict[str, Any], param_name: str) -> Optional[int]:
+    return _extract_arg(int, kwargs, param_name)
 
 
 @udf
-def _extract_float_tool_arg(tool_calls: dict[str, Any], func_name: str, param_name: str) -> Optional[float]:
-    return _extract_arg(float, tool_calls, func_name, param_name)
+def _extract_float_tool_arg(kwargs: dict[str, Any], param_name: str) -> Optional[float]:
+    return _extract_arg(float, kwargs, param_name)
 
 
 @udf
-def _extract_bool_tool_arg(tool_calls: dict[str, Any], func_name: str, param_name: str) -> Optional[bool]:
-    return _extract_arg(bool, tool_calls, func_name, param_name)
+def _extract_bool_tool_arg(kwargs: dict[str, Any], param_name: str) -> Optional[bool]:
+    return _extract_arg(bool, kwargs, param_name)
 
 
 T = TypeVar('T')
 
 
-def _extract_arg(
-    eval_fn: Callable[[Any], T], tool_calls: dict[str, Any], func_name: str, param_name: str
-) -> Optional[T]:
-    if func_name in tool_calls:
-        arguments = tool_calls[func_name]['args']
-        if param_name in arguments:
-            return eval_fn(arguments[param_name])
-        return None
+def _extract_arg(eval_fn: Callable[[Any], T], kwargs: dict[str, Any], param_name: str) -> Optional[T]:
+    if param_name in kwargs:
+        return eval_fn(kwargs[param_name])
     return None

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/functions/anthropic.py

@@ -213,9 +213,15 @@ def invoke_tools(tools: Tools, response: exprs.Expr) -> exprs.InlineDict:
 @pxt.udf
 def _anthropic_response_to_pxt_tool_calls(response: dict) -> Optional[dict]:
     anthropic_tool_calls = [r for r in response['content'] if r['type'] == 'tool_use']
-    if len(anthropic_tool_calls) > 0:
-        return {tool_call['name']: {'args': tool_call['input']} for tool_call in anthropic_tool_calls}
-    return None
+    if len(anthropic_tool_calls) == 0:
+        return None
+    pxt_tool_calls: dict[str, list[dict[str, Any]]] = {}
+    for tool_call in anthropic_tool_calls:
+        tool_name = tool_call['name']
+        if tool_name not in pxt_tool_calls:
+            pxt_tool_calls[tool_name] = []
+        pxt_tool_calls[tool_name].append({'args': tool_call['input']})
+    return pxt_tool_calls
 
 
 _T = TypeVar('_T')

{pixeltable-0.3.5 → pixeltable-0.3.6}/pixeltable/functions/openai.py

@@ -816,10 +816,13 @@ def _openai_response_to_pxt_tool_calls(response: dict) -> Optional[dict]:
     if 'tool_calls' not in response['choices'][0]['message'] or response['choices'][0]['message']['tool_calls'] is None:
         return None
     openai_tool_calls = response['choices'][0]['message']['tool_calls']
-    return {
-        tool_call['function']['name']: {'args': json.loads(tool_call['function']['arguments'])}
-        for tool_call in openai_tool_calls
-    }
+    pxt_tool_calls: dict[str, list[dict[str, Any]]] = {}
+    for tool_call in openai_tool_calls:
+        tool_name = tool_call['function']['name']
+        if tool_name not in pxt_tool_calls:
+            pxt_tool_calls[tool_name] = []
+        pxt_tool_calls[tool_name].append({'args': json.loads(tool_call['function']['arguments'])})
+    return pxt_tool_calls
 
 
 _T = TypeVar('_T')

{pixeltable-0.3.5 → pixeltable-0.3.6}/pyproject.toml

@@ -63,7 +63,7 @@ dependencies = [
     "pyiceberg (>=0.6.0)",
     "pixeltable-pgserver (==0.3.1)",
 ]
-version = "0.3.5"
+version = "0.3.6"
 
 [project.urls]
 homepage = "https://pixeltable.com/"