pixeltable 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

pixeltable/__init__.py

@@ -1,11 +1,10 @@
 from .catalog import Column, Table, InsertableTable, View
 from .dataframe import DataFrame
-from .datatransfer import Remote
-from .catalog import Column, Table, InsertableTable, View
-from .exceptions import Error, Error
+from .exceptions import Error
 from .exprs import RELATIVE_PATH_ROOT
-from .func import Function, udf, uda, Aggregator, expr_udf
-from .globals import *
+from .func import Function, udf, Aggregator, uda, expr_udf
+from .globals import init, create_table, create_view, get_table, move, drop_table, list_tables, create_dir, rm_dir, \
+    list_dirs, list_functions, get_path, configure_logging
 from .type_system import (
     ColumnType,
     StringType,
@@ -22,34 +21,17 @@ from .type_system import (
 )
 from .utils.help import help
 
-# noinspection PyUnresolvedReferences
 from . import functions, io, iterators
 from .__version__ import __version__, __version_tuple__
 
-__all__ = [
-    'DataFrame',
-    'Column',
-    'Table',
-    'InsertableTable',
-    'View',
-    'Error',
-    'ColumnType',
-    'StringType',
-    'IntType',
-    'FloatType',
-    'BoolType',
-    'TimestampType',
-    'JsonType',
-    'RELATIVE_PATH_ROOT',
-    'ArrayType',
-    'ImageType',
-    'VideoType',
-    'AudioType',
-    'DocumentType',
-    'Function',
-    'help',
-    'udf',
-    'Aggregator',
-    'uda',
-    'expr_udf',
-]
+# This is the safest / most maintainable way to do this: start with the default and "blacklist" stuff that
+# we don't want in there. (Using a "whitelist" is considerably harder to maintain.)
+
+__default_dir = set(symbol for symbol in dir() if not symbol.startswith('_'))
+__removed_symbols = {'catalog', 'dataframe', 'env', 'exceptions', 'exec', 'exprs', 'func', 'globals', 'index',
+                     'metadata', 'plan', 'type_system', 'utils'}
+__all__ = sorted(list(__default_dir - __removed_symbols))
+
+
+def __dir__():
+    return __all__

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.7"
-__version_tuple__ = (0, 2, 7)
+__version__ = "0.2.9"
+__version_tuple__ = (0, 2, 9)

pixeltable/catalog/catalog.py

@@ -39,7 +39,7 @@ class Catalog:
         # key: [id, version]
         # - mutable version of a table: version == None (even though TableVersion.version is set correctly)
         # - snapshot versions: records the version of the snapshot
-        self.tbl_versions: Dict[Tuple[UUID, int], TableVersion] = {}
+        self.tbl_versions: Dict[Tuple[UUID, Optional[int]], TableVersion] = {}
 
         self.tbls: Dict[UUID, Table] = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
         self.tbl_dependents: Dict[UUID, List[Table]] = {}

pixeltable/catalog/column.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import logging
-from typing import Optional, Union, Callable, Set
+from typing import Optional, Union, Callable, Any
+from uuid import UUID
 
 import sqlalchemy as sql
 
@@ -23,7 +24,7 @@ class Column:
             is_pk: bool = False, stored: Optional[bool] = None,
             col_id: Optional[int] = None, schema_version_add: Optional[int] = None,
             schema_version_drop: Optional[int] = None, sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
-            records_errors: Optional[bool] = None
+            records_errors: Optional[bool] = None, value_expr_dict: Optional[dict[str, Any]] = None,
     ):
         """Column constructor.
 
@@ -56,8 +57,9 @@ class Column:
         if col_type is None and computed_with is None:
             raise excs.Error(f'Column `{name}`: col_type is required if computed_with is not specified')
 
-        self.value_expr: Optional['Expr'] = None
+        self._value_expr: Optional['Expr'] = None
         self.compute_func: Optional[Callable] = None
+        self.value_expr_dict = value_expr_dict
         from pixeltable import exprs
         if computed_with is not None:
             value_expr = exprs.Expr.from_object(computed_with)
@@ -73,8 +75,8 @@ class Column:
                 # column name references and for that we need to wait until we're assigned to a Table
                 self.compute_func = computed_with
             else:
-                self.value_expr = value_expr.copy()
-                self.col_type = self.value_expr.col_type
+                self._value_expr = value_expr.copy()
+                self.col_type = self._value_expr.col_type
 
         if col_type is not None:
             self.col_type = col_type
@@ -87,11 +89,6 @@ class Column:
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
-        # stored_proxy may be set later if this is a non-stored column.
-        # if col1.stored_proxy == col2, then also col1 == col2.proxy_base.
-        self.stored_proxy: Optional[Column] = None
-        self.proxy_base: Optional[Column] = None
-
         self._records_errors = records_errors
 
         # column in the stored table for the values of this Column
@@ -105,12 +102,22 @@ class Column:
         from .table_version import TableVersion
         self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
 
-    def __hash__(self) -> int:
-        assert self.tbl is not None
-        return hash((self.tbl.id, self.id))
+    @property
+    def value_expr(self) -> Optional['Expr']:
+        """Instantiate value_expr on-demand"""
+        # TODO: instantiate expr in the c'tor and add an Expr.prepare() that can create additional state after the
+        # catalog has been fully loaded; that way, we encounter bugs in the serialization/deserialization logic earlier
+        if self.value_expr_dict is not None and self._value_expr is None:
+            from pixeltable import exprs
+            self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
+        return self._value_expr
+
+    def set_value_expr(self, value_expr: 'Expr') -> None:
+        self._value_expr = value_expr
+        self.value_expr_dict = None
 
     def check_value_expr(self) -> None:
-        assert self.value_expr is not None
+        assert self._value_expr is not None
         if self.stored == False and self.is_computed and self.has_window_fn_call():
             raise excs.Error(
                 f'Column {self.name}: stored={self.stored} not supported for columns computed with window functions:'
@@ -129,7 +136,7 @@ class Column:
 
     @property
     def is_computed(self) -> bool:
-        return self.compute_func is not None or self.value_expr is not None
+        return self.compute_func is not None or self._value_expr is not None or self.value_expr_dict is not None
 
     @property
     def is_stored(self) -> bool:
@@ -184,10 +191,15 @@ class Column:
     def __str__(self) -> str:
         return f'{self.name}: {self.col_type}'
 
+    def __hash__(self) -> int:
+        # TODO(aaron-siegel): This and __eq__ do not capture the table version. We need to rethink the Column
+        #     abstraction (perhaps separating out the version-dependent properties into a different abstraction).
+        assert self.tbl is not None
+        return hash((self.tbl.id, self.id))
+
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, Column):
             return False
         assert self.tbl is not None
         assert other.tbl is not None
         return self.tbl.id == other.tbl.id and self.id == other.id
-

pixeltable/catalog/dir.py

@@ -21,8 +21,8 @@ class Dir(SchemaObject):
     def display_name(cls) -> str:
         return 'directory'
 
-    def move(self, new_name: str, new_dir_id: UUID) -> None:
-        super().move(new_name, new_dir_id)
+    def _move(self, new_name: str, new_dir_id: UUID) -> None:
+        super()._move(new_name, new_dir_id)
         with Env.get().engine.begin() as conn:
             dir_md = schema.DirMd(name=new_name)
             conn.execute(

pixeltable/catalog/insertable_table.py

@@ -71,56 +71,6 @@ class InsertableTable(Table):
             self, rows: Optional[Iterable[dict[str, Any]]] = None, /, *, print_stats: bool = False,
             fail_on_exception: bool = True, **kwargs: Any
     ) -> UpdateStatus:
-        """Inserts rows into this table. There are two mutually exclusive call patterns:
-
-        To insert multiple rows at a time:
-        ``insert(rows: Iterable[dict[str, Any]], /, *, print_stats: bool = False, fail_on_exception: bool = True)``
-
-        To insert just a single row, you can use the more convenient syntax:
-        ``insert(*, print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any)``
-
-        Args:
-            rows: (if inserting multiple rows) A list of rows to insert, each of which is a dictionary mapping column
-                names to values.
-            kwargs: (if inserting a single row) Keyword-argument pairs representing column names and values.
-            print_stats: If ``True``, print statistics about the cost of computed columns.
-            fail_on_exception:
-                Determines how exceptions in computed columns and invalid media files (e.g., corrupt images)
-                are handled.
-                If ``False``, store error information (accessible as column properties 'errortype' and 'errormsg')
-                for those cases, but continue inserting rows.
-                If ``True``, raise an exception that aborts the insert.
-
-        Returns:
-            execution status
-
-        Raises:
-            Error: if a row does not match the table schema or contains values for computed columns
-
-        Examples:
-            Insert two rows into a table with three int columns ``a``, ``b``, and ``c``. Column ``c`` is nullable.
-
-            >>> tbl.insert([{'a': 1, 'b': 1, 'c': 1}, {'a': 2, 'b': 2}])
-
-            Insert a single row into a table with three int columns ``a``, ``b``, and ``c``.
-
-            >>> tbl.insert(a=1, b=1, c=1)
-        """
-        # The commented code is the intended implementation, with signature (*args, **kwargs).
-        # That signature cannot be used currently, due to a present limitation in mkdocs.
-        # See: https://github.com/mkdocstrings/mkdocstrings/issues/669
-
-        # print_stats = kwargs.pop('print_stats', False)
-        # fail_on_exception = kwargs.pop('fail_on_exception', True)
-        # if len(args) > 0:
-        #     # There's a positional argument; this means `rows` is expressed as a
-        #     # list of dicts (multi-insert)
-        #     rows = list(args[0])
-        # else:
-        #     # No positional argument; this means we're inserting a single row
-        #     # using kwargs syntax
-        #     rows = [kwargs]
-
         if rows is None:
             rows = [kwargs]
         else:
@@ -136,7 +86,7 @@ class InsertableTable(Table):
             if not isinstance(row, dict):
                 raise excs.Error('rows must be a list of dictionaries')
         self._validate_input_rows(rows)
-        result = self.tbl_version.insert(rows, print_stats=print_stats, fail_on_exception=fail_on_exception)
+        result = self._tbl_version.insert(rows, print_stats=print_stats, fail_on_exception=fail_on_exception)
 
         if result.num_excs == 0:
             cols_with_excs_str = ''
@@ -155,8 +105,8 @@ class InsertableTable(Table):
     def _validate_input_rows(self, rows: List[Dict[str, Any]]) -> None:
         """Verify that the input rows match the table schema"""
         valid_col_names = set(self.column_names())
-        reqd_col_names = set(self.tbl_version_path.tbl_version.get_required_col_names())
-        computed_col_names = set(self.tbl_version_path.tbl_version.get_computed_col_names())
+        reqd_col_names = set(self._tbl_version_path.tbl_version.get_required_col_names())
+        computed_col_names = set(self._tbl_version_path.tbl_version.get_computed_col_names())
         for row in rows:
             assert isinstance(row, dict)
             col_names = set(row.keys())
@@ -170,7 +120,7 @@ class InsertableTable(Table):
                     raise excs.Error(f'Value for computed column {col_name} in row {row}')
 
                 # validate data
-                col = self.tbl_version_path.get_column(col_name)
+                col = self._tbl_version_path.get_column(col_name)
                 try:
                     # basic sanity checks here
                     checked_val = col.col_type.create_literal(val)
@@ -199,7 +149,7 @@ class InsertableTable(Table):
         if where is not None:
             if not isinstance(where, Predicate):
                 raise excs.Error(f"'where' argument must be a Predicate, got {type(where)}")
-            analysis_info = Planner.analyze(self.tbl_version_path, where)
+            analysis_info = Planner.analyze(self._tbl_version_path, where)
             # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
             if analysis_info.filter is not None:
                 raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')

pixeltable/catalog/named_function.py

@@ -25,8 +25,8 @@ class NamedFunction(SchemaObject):
     def display_name(cls) -> str:
         return 'function'
 
-    def move(self, new_name: str, new_dir_id: UUID) -> None:
-        super().move(new_name, new_dir_id)
+    def _move(self, new_name: str, new_dir_id: UUID) -> None:
+        super()._move(new_name, new_dir_id)
         with Env.get().engine.begin() as conn:
             stmt = sql.text((
                 f"UPDATE {schema.Function.__table__} "

pixeltable/catalog/schema_object.py

@@ -14,7 +14,7 @@ class SchemaObject:
         self._name = name
         self._dir_id = dir_id
 
-    def get_id(self) -> UUID:
+    def _get_id(self) -> UUID:
         return self._id
 
     def get_name(self) -> str:
@@ -28,12 +28,7 @@ class SchemaObject:
         """
         pass
 
-    @property
-    def fqn(self) -> str:
-        return f'{self.parent_dir().fqn}.{self._name}'
-
-    def move(self, new_name: str, new_dir_id: UUID) -> None:
+    def _move(self, new_name: str, new_dir_id: UUID) -> None:
         """Subclasses need to override this to make the change persistent"""
         self._name = new_name
         self._dir_id = new_dir_id
-