pixeltable 0.3.8__py3-none-any.whl → 0.3.9__py3-none-any.whl

pixeltable/catalog/dir.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
 import dataclasses
+import datetime
+import json
 import logging
 from uuid import UUID
 
 import sqlalchemy as sql
+from sqlalchemy.dialects.postgresql import JSONB
 
 from pixeltable.env import Env
 from pixeltable.metadata import schema
@@ -26,6 +29,7 @@ class Dir(SchemaObject):
         dir_record = schema.Dir(parent_id=parent_id, md=dataclasses.asdict(dir_md))
         session.add(dir_record)
         session.flush()
+        # print(f'{datetime.datetime.now()} create dir {dir_record}')
         assert dir_record.id is not None
         assert isinstance(dir_record.id, UUID)
         dir = cls(dir_record.id, parent_id, name)
@@ -43,11 +47,16 @@ class Dir(SchemaObject):
         return super()._path()
 
     def _move(self, new_name: str, new_dir_id: UUID) -> None:
+        # print(
+        #     f'{datetime.datetime.now()} move dir name={self._name} parent={self._dir_id} new_name={new_name} new_dir_id={new_dir_id}'
+        # )
         super()._move(new_name, new_dir_id)
-        with Env.get().engine.begin() as conn:
-            dir_md = schema.DirMd(name=new_name, user=None, additional_md={})
-            conn.execute(
-                sql.update(schema.Dir.__table__)
-                .values({schema.Dir.parent_id: self._dir_id, schema.Dir.md: dataclasses.asdict(dir_md)})
-                .where(schema.Dir.id == self._id)
+        stmt = sql.text(
+            (
+                f'UPDATE {schema.Dir.__table__} '
+                f'SET {schema.Dir.parent_id.name} = :new_dir_id, '
+                f"    {schema.Dir.md.name}['name'] = :new_name "
+                f'WHERE {schema.Dir.id.name} = :id'
             )
+        )
+        Env.get().conn.execute(stmt, {'new_dir_id': new_dir_id, 'new_name': json.dumps(new_name), 'id': self._id})

pixeltable/catalog/path.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from typing import Iterator
 
 from pixeltable import exceptions as excs
 
@@ -55,5 +56,19 @@ class Path:
         is_prefix = self.components == other.components[: self.len]
         return is_prefix and (self.len == (other.len - 1) or not is_parent)
 
+    def ancestors(self) -> Iterator[Path]:
+        """
+        Return all ancestors of this path in top-down order including root.
+        If this path is for the root directory, which has no parent, then None is returned.
+        """
+        if self.is_root:
+            return
+        else:
+            for i in range(0, len(self.components)):
+                yield Path('.'.join(self.components[0:i]), empty_is_valid=True)
+
     def __str__(self) -> str:
         return '.'.join(self.components)
+
+    def __lt__(self, other: Path) -> bool:
+        return str(self) < str(other)

pixeltable/catalog/schema_object.py

@@ -2,7 +2,7 @@ from abc import abstractmethod
 from typing import TYPE_CHECKING, Any, Optional
 from uuid import UUID
 
-import pixeltable.env as env
+from pixeltable.env import Env
 
 if TYPE_CHECKING:
     from pixeltable import catalog
@@ -28,24 +28,19 @@ class SchemaObject:
         """Returns the parent directory of this schema object."""
         from .catalog import Catalog
 
-        with env.Env.get().begin_xact():
+        with Env.get().begin_xact():
             if self._dir_id is None:
                 return None
             return Catalog.get().get_dir(self._dir_id)
 
     def _path(self) -> str:
         """Returns the path to this schema object."""
-        with env.Env.get().begin_xact():
-            from .catalog import Catalog
+        from .catalog import Catalog
 
-            cat = Catalog.get()
-            dir_path = cat.get_dir_path(self._dir_id)
-            if dir_path == '':
-                # Either this is the root directory, with empty path, or its parent is the
-                # root directory. Either way, we return just the name.
-                return self._name
-            else:
-                return f'{dir_path}.{self._name}'
+        assert self._dir_id is not None
+        with Env.get().begin_xact():
+            path = Catalog.get().get_dir_path(self._dir_id)
+            return str(path.append(self._name))
 
     def get_metadata(self) -> dict[str, Any]:
         """Returns metadata associated with this schema object."""

pixeltable/catalog/table.py

@@ -171,8 +171,8 @@ class Table(SchemaObject):
 
     def _get_views(self, *, recursive: bool = True) -> list['Table']:
         cat = catalog.Catalog.get()
-        view_ids = cat.get_views(self._id)
-        views = [cat.get_tbl(id) for id in view_ids]
+        view_ids = cat.get_view_ids(self._id)
+        views = [cat.get_table_by_id(id) for id in view_ids]
         if recursive:
             views.extend([t for view in views for t in view._get_views(recursive=True)])
         return views
@@ -265,7 +265,7 @@ class Table(SchemaObject):
         if self._tbl_version_path.base is None:
             return None
         base_id = self._tbl_version_path.base.tbl_version.id
-        return catalog.Catalog.get().get_tbl(base_id)
+        return catalog.Catalog.get().get_table_by_id(base_id)
 
     @property
     def _bases(self) -> list['Table']:
@@ -369,11 +369,6 @@ class Table(SchemaObject):
             pd_rows.append(row)
         return pd.DataFrame(pd_rows)
 
-    def ensure_md_loaded(self) -> None:
-        """Ensure that table metadata is loaded."""
-        for col in self._tbl_version.get().cols_by_id.values():
-            _ = col.value_expr
-
     def describe(self) -> None:
         """
         Print the table schema.
@@ -387,13 +382,9 @@ class Table(SchemaObject):
             print(repr(self))
 
     def _drop(self) -> None:
-        cat = catalog.Catalog.get()
         self._check_is_dropped()
         self._tbl_version.get().drop()
         self._is_dropped = True
-        # update catalog
-        cat = catalog.Catalog.get()
-        cat.remove_tbl(self._id)
 
     # TODO Factor this out into a separate module.
     # The return type is unresolvable, but torch can't be imported since it's an optional dependency.

pixeltable/catalog/table_version.py

@@ -177,10 +177,6 @@ class TableVersion:
         # Init external stores (this needs to happen after the schema is created)
         self._init_external_stores(tbl_md)
 
-        # Force column metadata to load, in order to surface any invalid metadata now (as warnings)
-        for col in self.cols_by_id.values():
-            _ = col.value_expr
-
     def __hash__(self) -> int:
         return hash(self.id)
 
@@ -458,6 +454,11 @@ class TableVersion:
                 )
             )
 
+    def ensure_md_loaded(self) -> None:
+        """Ensure that table metadata is loaded."""
+        for col in self.cols_by_id.values():
+            _ = col.value_expr
+
     def _store_idx_name(self, idx_id: int) -> str:
         """Return name of index in the store, which needs to be globally unique"""
         return f'idx_{self.id.hex}_{idx_id}'

pixeltable/catalog/view.py

@@ -237,15 +237,11 @@ class View(Table):
         )
 
     def _drop(self) -> None:
-        cat = catalog.Catalog.get()
         if self._snapshot_only:
             # there is not TableVersion to drop
             self._check_is_dropped()
             self.is_dropped = True
             TableVersion.delete_md(self._id)
-            # update catalog
-            cat = catalog.Catalog.get()
-            cat.remove_tbl(self._id)
         else:
             super()._drop()
 

pixeltable/env.py

@@ -170,19 +170,25 @@ class Env:
         assert self._current_session is not None
         return self._current_session
 
+    def in_xact(self) -> bool:
+        return self._current_conn is not None
+
     @contextmanager
     def begin_xact(self) -> Iterator[sql.Connection]:
         """Return a context manager that yields a connection to the database. Idempotent."""
         if self._current_conn is None:
             assert self._current_session is None
-            with self.engine.begin() as conn, sql.orm.Session(conn) as session:
-                self._current_conn = conn
-                self._current_session = session
-                try:
+            try:
+                with self.engine.begin() as conn, sql.orm.Session(conn) as session:
+                    # TODO: remove print() once we're done with debugging the concurrent update behavior
+                    # print(f'{datetime.datetime.now()}: start xact')
+                    self._current_conn = conn
+                    self._current_session = session
                     yield conn
-                finally:
-                    self._current_session = None
-                    self._current_conn = None
+            finally:
+                self._current_session = None
+                self._current_conn = None
+                # print(f'{datetime.datetime.now()}: end xact')
         else:
             assert self._current_session is not None
             yield self._current_conn
@@ -391,7 +397,7 @@ class Env:
     def _create_engine(self, time_zone_name: Optional[str], echo: bool = False) -> None:
         connect_args = {} if time_zone_name is None else {'options': f'-c timezone={time_zone_name}'}
         self._sa_engine = sql.create_engine(
-            self.db_url, echo=echo, future=True, isolation_level='REPEATABLE READ', connect_args=connect_args
+            self.db_url, echo=echo, isolation_level='REPEATABLE READ', connect_args=connect_args
         )
         self._logger.info(f'Created SQLAlchemy engine at: {self.db_url}')
         with self.engine.begin() as conn:

pixeltable/exprs/column_ref.py

@@ -137,7 +137,7 @@ class ColumnRef(Expr):
         return self.col == other.col and self.perform_validation == other.perform_validation
 
     def _df(self) -> 'pxt.dataframe.DataFrame':
-        tbl = catalog.Catalog.get().get_tbl(self.col.tbl.id)
+        tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
         return tbl.select(self)
 
     def show(self, *args, **kwargs) -> 'pxt.dataframe.DataFrameResultSet':
@@ -165,7 +165,7 @@ class ColumnRef(Expr):
         return self._descriptors().to_html()
 
     def _descriptors(self) -> DescriptionHelper:
-        tbl = catalog.Catalog.get().get_tbl(self.col.tbl.id)
+        tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
         helper = DescriptionHelper()
         helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path()!r})')
         helper.append(tbl._col_descriptor([self.col.name]))

pixeltable/exprs/expr.py

@@ -90,14 +90,29 @@ class Expr(abc.ABC):
                 result = c_scope
         return result
 
-    def bind_rel_paths(self, mapper: Optional['exprs.JsonMapper'] = None) -> None:
+    def bind_rel_paths(self) -> None:
         """
         Binds relative JsonPaths to mapper.
         This needs to be done in a separate phase after __init__(), because RelativeJsonPath()(-1) cannot be resolved
         by the immediately containing JsonMapper during initialization.
         """
+        self._bind_rel_paths()
+        assert not self._has_relative_path, self._expr_tree()
+
+    def _bind_rel_paths(self, mapper: Optional['exprs.JsonMapper'] = None) -> None:
+        for c in self.components:
+            c._bind_rel_paths(mapper)
+
+    def _expr_tree(self) -> str:
+        """Returns a string representation of this expression as a multi-line tree. Useful for debugging."""
+        buf: list[str] = []
+        self._expr_tree_r(0, buf)
+        return '\n'.join(buf)
+
+    def _expr_tree_r(self, indent: int, buf: list[str]) -> None:
+        buf.append(f'{" " * indent}{type(self).__name__}: {self}'.replace('\n', '\\n'))
         for c in self.components:
-            c.bind_rel_paths(mapper)
+            c._expr_tree_r(indent + 2, buf)
 
     def default_column_name(self) -> Optional[str]:
         """
@@ -355,6 +370,10 @@ class Expr(abc.ABC):
         except StopIteration:
             return False
 
+    @property
+    def _has_relative_path(self) -> bool:
+        return any(c._has_relative_path for c in self.components)
+
     def tbl_ids(self) -> set[UUID]:
         """Returns table ids referenced by this expr."""
         from .column_ref import ColumnRef
@@ -514,7 +533,7 @@ class Expr(abc.ABC):
 
     @classmethod
     def _from_dict(cls, d: dict, components: list[Expr]) -> Self:
-        raise AssertionError('not implemented')
+        raise AssertionError(f'not implemented: {cls.__name__}')
 
     def isin(self, value_set: Any) -> 'exprs.InPredicate':
         from .in_predicate import InPredicate

pixeltable/exprs/function_call.py

@@ -360,10 +360,7 @@ class FunctionCall(Expr):
             return
         args, kwargs = args_kwargs
 
-        if isinstance(self.fn, func.CallableFunction) and not self.fn.is_batched:
-            # optimization: avoid additional level of indirection we'd get from calling Function.exec()
-            data_row[self.slot_idx] = self.fn.py_fn(*args, **kwargs)
-        elif self.is_window_fn_call:
+        if self.is_window_fn_call:
             assert isinstance(self.fn, func.AggregateFunction)
             agg_cls = self.fn.agg_class
             if self.has_group_by():

pixeltable/exprs/json_mapper.py

@@ -48,9 +48,9 @@ class JsonMapper(Expr):
         scope_anchor = ObjectRef(self.target_expr_scope, self)
         self.components.append(scope_anchor)
 
-    def bind_rel_paths(self, mapper: Optional[JsonMapper] = None) -> None:
-        self._src_expr.bind_rel_paths(mapper)
-        self._target_expr.bind_rel_paths(self)
+    def _bind_rel_paths(self, mapper: Optional[JsonMapper] = None) -> None:
+        self._src_expr._bind_rel_paths(mapper)
+        self._target_expr._bind_rel_paths(self)
         self.parent_mapper = mapper
         parent_scope = _GLOBAL_SCOPE if mapper is None else mapper.target_expr_scope
         self.target_expr_scope.parent = parent_scope

pixeltable/exprs/json_path.py

@@ -80,11 +80,16 @@ class JsonPath(Expr):
     def is_relative_path(self) -> bool:
         return self._anchor is None
 
-    def bind_rel_paths(self, mapper: Optional['JsonMapper'] = None) -> None:
-        if not self.is_relative_path():
-            return
-        # TODO: take scope_idx into account
-        self.set_anchor(mapper.scope_anchor)
+    @property
+    def _has_relative_path(self) -> bool:
+        return self.is_relative_path() or super()._has_relative_path
+
+    def _bind_rel_paths(self, mapper: Optional['JsonMapper'] = None) -> None:
+        if self.is_relative_path():
+            # TODO: take scope_idx into account
+            self.set_anchor(mapper.scope_anchor)
+        else:
+            self._anchor._bind_rel_paths(mapper)
 
     def __call__(self, *args: object, **kwargs: object) -> 'JsonPath':
         """

pixeltable/func/callable_function.py

@@ -8,6 +8,7 @@ from uuid import UUID
 import cloudpickle  # type: ignore[import-untyped]
 
 import pixeltable.exceptions as excs
+from pixeltable.utils.coroutine import run_coroutine_synchronously
 
 from .function import Function
 from .signature import Signature
@@ -93,13 +94,15 @@ class CallableFunction(Function):
             batched_kwargs = {k: [v] for k, v in kwargs.items() if k not in constant_param_names}
             result: list[Any]
             if inspect.iscoroutinefunction(self.py_fn):
-                result = asyncio.run(self.py_fn(*batched_args, **constant_kwargs, **batched_kwargs))
+                # TODO: This is temporary (see note in utils/coroutine.py)
+                result = run_coroutine_synchronously(self.py_fn(*batched_args, **constant_kwargs, **batched_kwargs))
             else:
                 result = self.py_fn(*batched_args, **constant_kwargs, **batched_kwargs)
             assert len(result) == 1
             return result[0]
         elif inspect.iscoroutinefunction(self.py_fn):
-            return asyncio.run(self.py_fn(*args, **kwargs))
+            # TODO: This is temporary (see note in utils/coroutine.py)
+            return run_coroutine_synchronously(self.py_fn(*args, **kwargs))
         else:
             return self.py_fn(*args, **kwargs)
 

pixeltable/func/query_template_function.py

@@ -17,8 +17,6 @@ class QueryTemplateFunction(Function):
 
     template_df: Optional['DataFrame']
     self_name: Optional[str]
-    # conn: Optional[sql.engine.Connection]
-    defaults: dict[str, exprs.Literal]
 
     @classmethod
     def create(
@@ -46,20 +44,6 @@ class QueryTemplateFunction(Function):
         self.self_name = name
         self.template_df = template_df
 
-        # if we're running as part of an ongoing update operation, we need to use the same connection, otherwise
-        # we end up with a deadlock
-        # TODO: figure out a more general way to make execution state available
-        # self.conn = None
-
-        # convert defaults to Literals
-        self.defaults = {}  # key: param name, value: default value converted to a Literal
-        param_types = self.template_df.parameters()
-        for param in [p for p in sig.parameters.values() if p.has_default()]:
-            assert param.name in param_types
-            param_type = param_types[param.name]
-            literal_default = exprs.Literal(param.default, col_type=param_type)
-            self.defaults[param.name] = literal_default
-
     def _update_as_overload_resolution(self, signature_idx: int) -> None:
         pass  # only one signature supported for QueryTemplateFunction
 
@@ -72,7 +56,11 @@ class QueryTemplateFunction(Function):
         bound_args = self.signature.py_signature.bind(*args, **kwargs).arguments
         # apply defaults, otherwise we might have Parameters left over
         bound_args.update(
-            {param_name: default for param_name, default in self.defaults.items() if param_name not in bound_args}
+            {
+                param.name: param.default
+                for param in self.signature.parameters.values()
+                if param.has_default() and param.name not in bound_args
+            }
         )
         bound_df = self.template_df.bind(bound_args)
         result = await bound_df._acollect()
@@ -87,7 +75,7 @@ class QueryTemplateFunction(Function):
         return self.self_name
 
     def _as_dict(self) -> dict:
-        return {'name': self.name, 'signature': self.signatures[0].as_dict(), 'df': self.template_df.as_dict()}
+        return {'name': self.name, 'signature': self.signature.as_dict(), 'df': self.template_df.as_dict()}
 
     @classmethod
     def _from_dict(cls, d: dict) -> Function: