pixeltable 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

pixeltable/catalog/dir.py

@@ -34,8 +34,7 @@ class Dir(SchemaObject):
         dir = cls(dir_record.id, parent_id, name)
         return dir
 
-    @classmethod
-    def _display_name(cls) -> str:
+    def _display_name(self) -> str:
         return 'directory'
 
     def _path(self) -> str:

pixeltable/catalog/insertable_table.py

@@ -54,8 +54,8 @@ class InsertableTable(Table):
         super().__init__(tbl_version.id, dir_id, tbl_version.get().name, tbl_version_path)
         self._tbl_version = tbl_version
 
-    @classmethod
-    def _display_name(cls) -> str:
+    def _display_name(self) -> str:
+        assert not self._tbl_version_path.is_replica()
         return 'table'
 
     @classmethod
@@ -75,10 +75,10 @@ class InsertableTable(Table):
         column_names = [col.name for col in columns]
         for pk_col in primary_key:
             if pk_col not in column_names:
-                raise excs.Error(f'Primary key column {pk_col} not found in table schema')
+                raise excs.Error(f'Primary key column {pk_col!r} not found in table schema.')
             col = columns[column_names.index(pk_col)]
             if col.col_type.nullable:
-                raise excs.Error(f'Primary key column {pk_col} cannot be nullable')
+                raise excs.Error(f'Primary key column {pk_col!r} cannot be nullable.')
             col.is_pk = True
 
         _, tbl_version = TableVersion.create(
@@ -101,8 +101,8 @@ class InsertableTable(Table):
             tbl_version.insert(None, df, fail_on_exception=True)
         session.commit()
 
-        _logger.info(f'Created table `{name}`, id={tbl_version.id}')
-        Env.get().console_logger.info(f'Created table `{name}`.')
+        _logger.info(f'Created table {name!r}, id={tbl_version.id}')
+        Env.get().console_logger.info(f'Created table {name!r}.')
         return tbl
 
     def _get_metadata(self) -> dict[str, Any]:
@@ -204,9 +204,9 @@ class InsertableTable(Table):
 
             for col_name, val in row.items():
                 if col_name not in valid_col_names:
-                    raise excs.Error(f'Unknown column name {col_name} in row {row}')
+                    raise excs.Error(f'Unknown column name {col_name!r} in row {row}')
                 if col_name in computed_col_names:
-                    raise excs.Error(f'Value for computed column {col_name} in row {row}')
+                    raise excs.Error(f'Value for computed column {col_name!r} in row {row}')
 
                 # validate data
                 col = self._tbl_version_path.get_column(col_name)
@@ -246,4 +246,4 @@ class InsertableTable(Table):
         return []
 
     def _table_descriptor(self) -> str:
-        return f'Table {self._path()!r}'
+        return self._display_str()

pixeltable/catalog/path.py

@@ -1,20 +1,57 @@
 from __future__ import annotations
 
 import logging
-from typing import Iterator
+from typing import Optional
 
 from pixeltable import exceptions as excs
 
-from .globals import is_valid_path
+from .globals import is_valid_identifier
 
 _logger = logging.getLogger('pixeltable')
 
 
 class Path:
-    def __init__(self, path: str, empty_is_valid: bool = False, allow_system_paths: bool = False):
-        if not is_valid_path(path, empty_is_valid, allow_system_paths):
-            raise excs.Error(f"Invalid path format: '{path}'")
-        self.components = path.split('.')
+    components: list[str]
+    version: Optional[int]
+
+    def __init__(self, components: list[str], version: Optional[int] = None) -> None:
+        assert len(components) > 0
+        self.components = components
+        self.version = version
+
+    @classmethod
+    def parse(
+        cls,
+        path: str,
+        allow_empty_path: bool = False,
+        allow_system_path: bool = False,
+        allow_versioned_path: bool = False,
+    ) -> Path:
+        components: list[str]
+        version: Optional[int]
+        if ':' in path:
+            parts = path.split(':')
+            if len(parts) != 2:
+                raise excs.Error(f'Invalid path: {path}')
+            try:
+                components = parts[0].split('.')
+                version = int(parts[1])
+            except ValueError:
+                raise excs.Error(f'Invalid path: {path}') from None
+        else:
+            components = path.split('.')
+            version = None
+
+        if components == [''] and not allow_empty_path:
+            raise excs.Error(f'Invalid path: {path}')
+
+        if components != [''] and not all(is_valid_identifier(c, allow_system_path) for c in components):
+            raise excs.Error(f'Invalid path: {path}')
+
+        if version is not None and not allow_versioned_path:
+            raise excs.Error(f'Versioned path not allowed here: {path}')
+
+        return Path(components, version)
 
     @property
     def len(self) -> int:
@@ -22,7 +59,6 @@ class Path:
 
     @property
     def name(self) -> str:
-        assert len(self.components) > 0
         return self.components[-1]
 
     @property
@@ -36,18 +72,15 @@ class Path:
     @property
     def parent(self) -> Path:
         if len(self.components) == 1:
-            if self.is_root:
-                return self
-            else:
-                return Path('', empty_is_valid=True, allow_system_paths=True)
+            return ROOT_PATH  # Includes the case of the root path, which is its own parent.
         else:
-            return Path('.'.join(self.components[:-1]), allow_system_paths=True)
+            return Path(self.components[:-1])
 
     def append(self, name: str) -> Path:
         if self.is_root:
-            return Path(name, allow_system_paths=True)
+            return Path([name])
         else:
-            return Path(f'{self}.{name}', allow_system_paths=True)
+            return Path([*self.components, name])
 
     def is_ancestor(self, other: Path, is_parent: bool = False) -> bool:
         """
@@ -60,22 +93,25 @@ class Path:
         is_prefix = self.components == other.components[: self.len]
         return is_prefix and (self.len == (other.len - 1) or not is_parent)
 
-    def ancestors(self) -> Iterator[Path]:
+    def ancestors(self) -> list[Path]:
         """
-        Return all ancestors of this path in top-down order including root.
+        Return all proper ancestors of this path in top-down order including root.
         If this path is for the root directory, which has no parent, then None is returned.
         """
         if self.is_root:
-            return
+            return []
         else:
-            for i in range(0, len(self.components)):
-                yield Path('.'.join(self.components[0:i]), empty_is_valid=True)
+            return [Path(self.components[:i]) if i > 0 else ROOT_PATH for i in range(len(self.components))]
 
     def __repr__(self) -> str:
         return repr(str(self))
 
     def __str__(self) -> str:
-        return '.'.join(self.components)
+        base = '.'.join(self.components)
+        if self.version is not None:
+            return f'{base}:{self.version}'
+        else:
+            return base
 
     def __eq__(self, other: object) -> bool:
         return isinstance(other, Path) and str(self) == str(other)
@@ -85,3 +121,6 @@ class Path:
 
     def __lt__(self, other: Path) -> bool:
         return str(self) < str(other)
+
+
+ROOT_PATH = Path([''])

pixeltable/catalog/schema_object.py

@@ -18,6 +18,7 @@ class SchemaObject:
 
     def __init__(self, obj_id: UUID, name: str, dir_id: Optional[UUID]):
         # make these private so they don't collide with column names (id and name are fairly common)
+        assert dir_id is None or isinstance(dir_id, UUID), type(dir_id)
         self._id = obj_id
         self._name = name
         self._dir_id = dir_id
@@ -42,22 +43,27 @@ class SchemaObject:
 
     def get_metadata(self) -> dict[str, Any]:
         """Returns metadata associated with this schema object."""
-        from pixeltable.catalog import Catalog
+        from pixeltable.catalog import retry_loop
 
-        with Catalog.get().begin_xact(for_write=False):
+        @retry_loop(for_write=False)
+        def op() -> dict[str, Any]:
             return self._get_metadata()
 
+        return op()
+
     def _get_metadata(self) -> dict[str, Any]:
         return {'name': self._name, 'path': self._path()}
 
-    @classmethod
     @abstractmethod
-    def _display_name(cls) -> str:
+    def _display_name(self) -> str:
         """
         Return name displayed in error messages.
         """
         pass
 
+    def _display_str(self) -> str:
+        return f'{self._display_name()} {self._path()!r}'
+
     def _move(self, new_name: str, new_dir_id: UUID) -> None:
         """Subclasses need to override this to make the change persistent"""
         self._name = new_name

pixeltable/catalog/table.py

@@ -89,6 +89,8 @@ class Table(SchemaObject):
 
                 ```python
                 {
+                    'name': 'my_table',
+                    'path': 'my_dir.my_subdir.my_table',
                     'base': None,  # If this is a view or snapshot, will contain the name of its base table
                     'schema': {
                         'col1': StringType(),
@@ -96,6 +98,7 @@ class Table(SchemaObject):
                     },
                     'is_replica': False,
                     'version': 22,
+                    'version_created': datetime.datetime(...),
                     'schema_version': 1,
                     'comment': '',
                     'num_retained_versions': 10,
@@ -112,6 +115,9 @@ class Table(SchemaObject):
         md['schema'] = self._get_schema()
         md['is_replica'] = self._tbl_version_path.is_replica()
         md['version'] = self._get_version()
+        md['version_created'] = datetime.datetime.fromtimestamp(
+            self._tbl_version_path.tbl_version.get().created_at, tz=datetime.timezone.utc
+        )
         md['schema_version'] = self._tbl_version_path.schema_version()
         md['comment'] = self._get_comment()
         md['num_retained_versions'] = self._get_num_retained_versions()
@@ -147,11 +153,15 @@ class Table(SchemaObject):
         Returns:
             A list of view paths.
         """
-        from pixeltable.catalog import Catalog
+        from pixeltable.catalog import retry_loop
 
-        with Catalog.get().begin_xact(for_write=False):
+        # we need retry_loop() here, because we end up loading Tables for the views
+        @retry_loop(tbl=self._tbl_version_path, for_write=False)
+        def op() -> list[str]:
             return [t._path() for t in self._get_views(recursive=recursive)]
 
+        return op()
+
     def _get_views(self, *, recursive: bool = True, include_snapshots: bool = True) -> list['Table']:
         cat = catalog.Catalog.get()
         view_ids = cat.get_view_ids(self._id)
@@ -178,7 +188,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=False):
             return self._df().select(*items, **named_items)
 
     def where(self, pred: 'exprs.Expr') -> 'pxt.DataFrame':
@@ -188,7 +198,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=False):
             return self._df().where(pred)
 
     def join(
@@ -201,7 +211,7 @@ class Table(SchemaObject):
         """Join this table with another table."""
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=False):
             return self._df().join(other, on=on, how=how)
 
     def order_by(self, *items: 'exprs.Expr', asc: bool = True) -> 'pxt.DataFrame':
@@ -211,7 +221,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=False):
             return self._df().order_by(*items, asc=asc)
 
     def group_by(self, *items: 'exprs.Expr') -> 'pxt.DataFrame':
@@ -221,7 +231,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=False):
             return self._df().group_by(*items)
 
     def distinct(self) -> 'pxt.DataFrame':
@@ -277,10 +287,7 @@ class Table(SchemaObject):
         return {c.name: c.col_type for c in self._tbl_version_path.columns()}
 
     def get_base_table(self) -> Optional['Table']:
-        from pixeltable.catalog import Catalog
-
-        with Catalog.get().begin_xact(for_write=False):
-            return self._get_base_table()
+        return self._get_base_table()
 
     @abc.abstractmethod
     def _get_base_table(self) -> Optional['Table']:
@@ -321,7 +328,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=False):
             helper = DescriptionHelper()
             helper.append(self._table_descriptor())
             helper.append(self._col_descriptor())
@@ -492,8 +499,7 @@ class Table(SchemaObject):
 
         # lock_mutable_tree=True: we might end up having to drop existing columns, which requires locking the tree
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot add column to a snapshot.')
+            self.__check_mutable('add columns to')
             col_schema = {
                 col_name: {'type': ts.ColumnType.normalize_type(spec, nullable_default=True, allow_builtin_types=False)}
                 for col_name, spec in schema.items()
@@ -553,24 +559,18 @@ class Table(SchemaObject):
 
             >>> tbl.add_columns({'new_col': pxt.Int})
         """
-        from pixeltable.catalog import Catalog
-
-        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            # verify kwargs
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot add column to a snapshot.')
-            # verify kwargs and construct column schema dict
-            if len(kwargs) != 1:
-                raise excs.Error(
-                    f'add_column() requires exactly one keyword argument of the form "col_name=col_type"; '
-                    f'got {len(kwargs)} instead ({", ".join(kwargs.keys())})'
-                )
-            col_type = next(iter(kwargs.values()))
-            if not isinstance(col_type, (ts.ColumnType, type, _GenericAlias)):
-                raise excs.Error(
-                    'The argument to add_column() must be a type; did you intend to use add_computed_column() instead?'
-                )
-            return self.add_columns(kwargs, if_exists=if_exists)
+        # verify kwargs and construct column schema dict
+        if len(kwargs) != 1:
+            raise excs.Error(
+                f'add_column() requires exactly one keyword argument of the form "col_name=col_type"; '
+                f'got {len(kwargs)} instead ({", ".join(kwargs.keys())})'
+            )
+        col_type = next(iter(kwargs.values()))
+        if not isinstance(col_type, (ts.ColumnType, type, _GenericAlias)):
+            raise excs.Error(
+                'The argument to add_column() must be a type; did you intend to use add_computed_column() instead?'
+            )
+        return self.add_columns(kwargs, if_exists=if_exists)
 
     def add_computed_column(
         self,
@@ -622,8 +622,7 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot add column to a snapshot.')
+            self.__check_mutable('add columns to')
             if len(kwargs) != 1:
                 raise excs.Error(
                     f'add_computed_column() requires exactly one keyword argument of the form '
@@ -808,10 +807,10 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         cat = Catalog.get()
+
         # lock_mutable_tree=True: we need to be able to see whether any transitive view has column dependents
         with cat.begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot drop column from a snapshot.')
+            self.__check_mutable('drop columns from')
             col: Column = None
             if_not_exists_ = IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
 
@@ -835,7 +834,7 @@ class Table(SchemaObject):
             dependent_user_cols = [c for c in cat.get_column_dependents(col.tbl.id, col.id) if c.name is not None]
             if len(dependent_user_cols) > 0:
                 raise excs.Error(
-                    f'Cannot drop column `{col.name}` because the following columns depend on it:\n'
+                    f'Cannot drop column {col.name!r} because the following columns depend on it:\n'
                     f'{", ".join(c.name for c in dependent_user_cols)}'
                 )
 
@@ -989,8 +988,7 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot add an index to a snapshot')
+            self.__check_mutable('add an index to')
             col = self._resolve_column_parameter(column)
 
             if idx_name is not None and idx_name in self._tbl_version.get().idxs_by_name:
@@ -1174,8 +1172,7 @@ class Table(SchemaObject):
     ) -> None:
         from pixeltable.catalog import Catalog
 
-        if self._tbl_version_path.is_snapshot():
-            raise excs.Error('Cannot drop an index from a snapshot')
+        self.__check_mutable('drop an index from')
         assert (col is None) != (idx_name is None)
 
         if idx_name is not None:
@@ -1347,8 +1344,7 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot update a snapshot')
+            self.__check_mutable('update')
             result = self._tbl_version.get().update(value_spec, where, cascade)
             FileCache.get().emit_eviction_warnings()
             return result
@@ -1387,8 +1383,7 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot update a snapshot')
+            self.__check_mutable('update')
             rows = list(rows)
 
             row_updates: list[dict[Column, exprs.Expr]] = []
@@ -1456,8 +1451,7 @@ class Table(SchemaObject):
         cat = Catalog.get()
         # lock_mutable_tree=True: we need to be able to see whether any transitive view has column dependents
         with cat.begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot recompute columns of a snapshot.')
+            self.__check_mutable('recompute columns of')
             if len(columns) == 0:
                 raise excs.Error('At least one column must be specified to recompute')
             if errors_only and len(columns) > 1:
@@ -1514,8 +1508,7 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error('Cannot revert a snapshot')
+            self.__check_mutable('revert')
             self._tbl_version.get().revert()
             # remove cached md in order to force a reload on the next operation
             self._tbl_version_path.clear_cached_md()
@@ -1530,8 +1523,7 @@ class Table(SchemaObject):
         from pixeltable.catalog import Catalog
 
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=False):
-            if self._tbl_version_path.is_snapshot():
-                raise excs.Error(f'Table `{self._name}` is a snapshot, so it cannot be linked to an external store.')
+            self.__check_mutable('link an external store to')
             if store.name in self.external_stores():
                 raise excs.Error(f'Table `{self._name}` already has an external store with that name: {store.name}')
             _logger.info(f'Linking external store `{store.name}` to table `{self._name}`')
@@ -1560,7 +1552,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        if self._tbl_version_path.is_snapshot():
+        if not self._tbl_version_path.is_mutable():
             return
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=False):
             all_stores = self.external_stores()
@@ -1600,7 +1592,7 @@ class Table(SchemaObject):
         """
         from pixeltable.catalog import Catalog
 
-        if self._tbl_version_path.is_snapshot():
+        if not self._tbl_version_path.is_mutable():
             return UpdateStatus()
         # we lock the entire tree starting at the root base table in order to ensure that all synced columns can
         # have their updates propagated down the tree
@@ -1711,3 +1703,9 @@ class Table(SchemaObject):
             report_lines.append(report_line)
 
         return pxt.dataframe.DataFrameResultSet(report_lines, self._REPORT_SCHEMA)
+
+    def __check_mutable(self, op_descr: str) -> None:
+        if self._tbl_version_path.is_snapshot():
+            raise excs.Error(f'{self._display_str()}: Cannot {op_descr} a snapshot.')
+        if self._tbl_version_path.is_replica():
+            raise excs.Error(f'{self._display_str()}: Cannot {op_descr} a {self._display_name()}.')