pixeltable 0.3.14__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pixeltable might be problematic. Click here for more details.

Files changed (50) hide show
  1. pixeltable/__version__.py +2 -2
  2. pixeltable/catalog/catalog.py +292 -105
  3. pixeltable/catalog/column.py +10 -8
  4. pixeltable/catalog/dir.py +1 -2
  5. pixeltable/catalog/insertable_table.py +25 -20
  6. pixeltable/catalog/schema_object.py +3 -6
  7. pixeltable/catalog/table.py +245 -189
  8. pixeltable/catalog/table_version.py +319 -201
  9. pixeltable/catalog/table_version_handle.py +15 -2
  10. pixeltable/catalog/table_version_path.py +60 -21
  11. pixeltable/catalog/view.py +14 -5
  12. pixeltable/dataframe.py +11 -9
  13. pixeltable/env.py +2 -4
  14. pixeltable/exec/in_memory_data_node.py +1 -1
  15. pixeltable/exec/sql_node.py +20 -11
  16. pixeltable/exprs/column_property_ref.py +15 -6
  17. pixeltable/exprs/column_ref.py +32 -11
  18. pixeltable/exprs/comparison.py +1 -1
  19. pixeltable/exprs/row_builder.py +4 -6
  20. pixeltable/exprs/rowid_ref.py +8 -0
  21. pixeltable/exprs/similarity_expr.py +1 -0
  22. pixeltable/func/query_template_function.py +1 -1
  23. pixeltable/functions/gemini.py +166 -33
  24. pixeltable/functions/math.py +63 -0
  25. pixeltable/functions/string.py +212 -58
  26. pixeltable/globals.py +7 -4
  27. pixeltable/index/base.py +5 -0
  28. pixeltable/index/btree.py +5 -0
  29. pixeltable/index/embedding_index.py +5 -0
  30. pixeltable/io/external_store.py +8 -29
  31. pixeltable/io/label_studio.py +1 -1
  32. pixeltable/io/parquet.py +4 -4
  33. pixeltable/io/table_data_conduit.py +0 -31
  34. pixeltable/metadata/__init__.py +1 -1
  35. pixeltable/metadata/converters/convert_13.py +2 -2
  36. pixeltable/metadata/converters/convert_30.py +6 -11
  37. pixeltable/metadata/converters/convert_35.py +9 -0
  38. pixeltable/metadata/converters/util.py +3 -9
  39. pixeltable/metadata/notes.py +1 -0
  40. pixeltable/metadata/schema.py +5 -1
  41. pixeltable/plan.py +4 -4
  42. pixeltable/share/packager.py +207 -15
  43. pixeltable/share/publish.py +2 -2
  44. pixeltable/store.py +31 -13
  45. pixeltable/utils/dbms.py +1 -1
  46. {pixeltable-0.3.14.dist-info → pixeltable-0.4.0rc1.dist-info}/METADATA +1 -1
  47. {pixeltable-0.3.14.dist-info → pixeltable-0.4.0rc1.dist-info}/RECORD +50 -49
  48. {pixeltable-0.3.14.dist-info → pixeltable-0.4.0rc1.dist-info}/LICENSE +0 -0
  49. {pixeltable-0.3.14.dist-info → pixeltable-0.4.0rc1.dist-info}/WHEEL +0 -0
  50. {pixeltable-0.3.14.dist-info → pixeltable-0.4.0rc1.dist-info}/entry_points.txt +0 -0
@@ -4,11 +4,11 @@ import abc
4
4
  import builtins
5
5
  import json
6
6
  import logging
7
+ from keyword import iskeyword as is_python_keyword
7
8
  from pathlib import Path
8
9
  from typing import TYPE_CHECKING, Any, Iterable, Literal, Optional, Union, overload
9
10
 
10
11
  from typing import _GenericAlias # type: ignore[attr-defined] # isort: skip
11
- from keyword import iskeyword as is_python_keyword
12
12
  from uuid import UUID
13
13
 
14
14
  import pandas as pd
@@ -16,7 +16,6 @@ import sqlalchemy as sql
16
16
 
17
17
  import pixeltable as pxt
18
18
  from pixeltable import catalog, env, exceptions as excs, exprs, index, type_system as ts
19
- from pixeltable.env import Env
20
19
  from pixeltable.metadata import schema
21
20
 
22
21
  from ..exprs import ColumnRef
@@ -62,11 +61,6 @@ class Table(SchemaObject):
62
61
  self._is_dropped = False
63
62
  self.__tbl_version_path = tbl_version_path
64
63
 
65
- # @property
66
- # def _has_dependents(self) -> bool:
67
- # """Returns True if this table has any dependent views, or snapshots."""
68
- # return len(self._get_views(recursive=False)) > 0
69
-
70
64
  def _move(self, new_name: str, new_dir_id: UUID) -> None:
71
65
  self._check_is_dropped()
72
66
  super()._move(new_name, new_dir_id)
@@ -106,10 +100,12 @@ class Table(SchemaObject):
106
100
  }
107
101
  ```
108
102
  """
109
- self._check_is_dropped()
110
- with env.Env.get().begin_xact():
103
+ from pixeltable.catalog import Catalog
104
+
105
+ with Catalog.get().begin_xact(for_write=False):
106
+ self._check_is_dropped()
111
107
  md = super().get_metadata()
112
- md['base'] = self._base_table._path if self._base_table is not None else None
108
+ md['base'] = self._base_table._path() if self._base_table is not None else None
113
109
  md['schema'] = self._schema
114
110
  md['is_replica'] = self._tbl_version.get().is_replica
115
111
  md['version'] = self._version
@@ -163,9 +159,11 @@ class Table(SchemaObject):
163
159
  Returns:
164
160
  A list of view paths.
165
161
  """
166
- self._check_is_dropped()
167
- with env.Env.get().begin_xact():
168
- return [t._path for t in self._get_views(recursive=recursive)]
162
+ from pixeltable.catalog import Catalog
163
+
164
+ with Catalog.get().begin_xact(for_write=False):
165
+ self._check_is_dropped()
166
+ return [t._path() for t in self._get_views(recursive=recursive)]
169
167
 
170
168
  def _get_views(self, *, recursive: bool = True) -> list['Table']:
171
169
  cat = catalog.Catalog.get()
@@ -187,14 +185,20 @@ class Table(SchemaObject):
187
185
 
188
186
  See [`DataFrame.select`][pixeltable.DataFrame.select] for more details.
189
187
  """
190
- return self._df().select(*items, **named_items)
188
+ from pixeltable.catalog import Catalog
189
+
190
+ with Catalog.get().begin_xact(for_write=False):
191
+ return self._df().select(*items, **named_items)
191
192
 
192
193
  def where(self, pred: 'exprs.Expr') -> 'pxt.DataFrame':
193
194
  """Filter rows from this table based on the expression.
194
195
 
195
196
  See [`DataFrame.where`][pixeltable.DataFrame.where] for more details.
196
197
  """
197
- return self._df().where(pred)
198
+ from pixeltable.catalog import Catalog
199
+
200
+ with Catalog.get().begin_xact(for_write=False):
201
+ return self._df().where(pred)
198
202
 
199
203
  def join(
200
204
  self,
@@ -204,21 +208,30 @@ class Table(SchemaObject):
204
208
  how: 'pixeltable.plan.JoinType.LiteralType' = 'inner',
205
209
  ) -> 'pxt.DataFrame':
206
210
  """Join this table with another table."""
207
- return self._df().join(other, on=on, how=how)
211
+ from pixeltable.catalog import Catalog
212
+
213
+ with Catalog.get().begin_xact(for_write=False):
214
+ return self._df().join(other, on=on, how=how)
208
215
 
209
216
  def order_by(self, *items: 'exprs.Expr', asc: bool = True) -> 'pxt.DataFrame':
210
217
  """Order the rows of this table based on the expression.
211
218
 
212
219
  See [`DataFrame.order_by`][pixeltable.DataFrame.order_by] for more details.
213
220
  """
214
- return self._df().order_by(*items, asc=asc)
221
+ from pixeltable.catalog import Catalog
222
+
223
+ with Catalog.get().begin_xact(for_write=False):
224
+ return self._df().order_by(*items, asc=asc)
215
225
 
216
226
  def group_by(self, *items: 'exprs.Expr') -> 'pxt.DataFrame':
217
227
  """Group the rows of this table based on the expression.
218
228
 
219
229
  See [`DataFrame.group_by`][pixeltable.DataFrame.group_by] for more details.
220
230
  """
221
- return self._df().group_by(*items)
231
+ from pixeltable.catalog import Catalog
232
+
233
+ with Catalog.get().begin_xact(for_write=False):
234
+ return self._df().group_by(*items)
222
235
 
223
236
  def distinct(self) -> 'pxt.DataFrame':
224
237
  """Remove duplicate rows from table."""
@@ -305,18 +318,21 @@ class Table(SchemaObject):
305
318
  """
306
319
  Constructs a list of descriptors for this table that can be pretty-printed.
307
320
  """
308
- helper = DescriptionHelper()
309
- helper.append(self._table_descriptor())
310
- helper.append(self._col_descriptor())
311
- idxs = self._index_descriptor()
312
- if not idxs.empty:
313
- helper.append(idxs)
314
- stores = self._external_store_descriptor()
315
- if not stores.empty:
316
- helper.append(stores)
317
- if self._comment:
318
- helper.append(f'COMMENT: {self._comment}')
319
- return helper
321
+ from pixeltable.catalog import Catalog
322
+
323
+ with Catalog.get().begin_xact(for_write=False):
324
+ helper = DescriptionHelper()
325
+ helper.append(self._table_descriptor())
326
+ helper.append(self._col_descriptor())
327
+ idxs = self._index_descriptor()
328
+ if not idxs.empty:
329
+ helper.append(idxs)
330
+ stores = self._external_store_descriptor()
331
+ if not stores.empty:
332
+ helper.append(stores)
333
+ if self._comment:
334
+ helper.append(f'COMMENT: {self._comment}')
335
+ return helper
320
336
 
321
337
  def _col_descriptor(self, columns: Optional[list[str]] = None) -> pd.DataFrame:
322
338
  return pd.DataFrame(
@@ -473,15 +489,17 @@ class Table(SchemaObject):
473
489
  ... }
474
490
  ... tbl.add_columns(schema)
475
491
  """
476
- self._check_is_dropped()
477
- if self.get_metadata()['is_snapshot']:
478
- raise excs.Error('Cannot add column to a snapshot.')
479
- col_schema = {
480
- col_name: {'type': ts.ColumnType.normalize_type(spec, nullable_default=True, allow_builtin_types=False)}
481
- for col_name, spec in schema.items()
482
- }
483
-
484
- with Env.get().begin_xact():
492
+ from pixeltable.catalog import Catalog
493
+
494
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
495
+ self._check_is_dropped()
496
+ if self.get_metadata()['is_snapshot']:
497
+ raise excs.Error('Cannot add column to a snapshot.')
498
+ col_schema = {
499
+ col_name: {'type': ts.ColumnType.normalize_type(spec, nullable_default=True, allow_builtin_types=False)}
500
+ for col_name, spec in schema.items()
501
+ }
502
+
485
503
  # handle existing columns based on if_exists parameter
486
504
  cols_to_ignore = self._ignore_or_drop_existing_columns(
487
505
  list(col_schema.keys()), IfExistsParam.validated(if_exists, 'if_exists')
@@ -534,22 +552,25 @@ class Table(SchemaObject):
534
552
 
535
553
  >>> tbl.add_columns({'new_col': pxt.Int})
536
554
  """
537
- self._check_is_dropped()
538
- # verify kwargs
539
- if self._tbl_version.get().is_snapshot:
540
- raise excs.Error('Cannot add column to a snapshot.')
541
- # verify kwargs and construct column schema dict
542
- if len(kwargs) != 1:
543
- raise excs.Error(
544
- f'add_column() requires exactly one keyword argument of the form "col_name=col_type"; '
545
- f'got {len(kwargs)} instead ({", ".join(kwargs.keys())})'
546
- )
547
- col_type = next(iter(kwargs.values()))
548
- if not isinstance(col_type, (ts.ColumnType, type, _GenericAlias)):
549
- raise excs.Error(
550
- 'The argument to add_column() must be a type; did you intend to use add_computed_column() instead?'
551
- )
552
- return self.add_columns(kwargs, if_exists=if_exists)
555
+ from pixeltable.catalog import Catalog
556
+
557
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
558
+ self._check_is_dropped()
559
+ # verify kwargs
560
+ if self._tbl_version.get().is_snapshot:
561
+ raise excs.Error('Cannot add column to a snapshot.')
562
+ # verify kwargs and construct column schema dict
563
+ if len(kwargs) != 1:
564
+ raise excs.Error(
565
+ f'add_column() requires exactly one keyword argument of the form "col_name=col_type"; '
566
+ f'got {len(kwargs)} instead ({", ".join(kwargs.keys())})'
567
+ )
568
+ col_type = next(iter(kwargs.values()))
569
+ if not isinstance(col_type, (ts.ColumnType, type, _GenericAlias)):
570
+ raise excs.Error(
571
+ 'The argument to add_column() must be a type; did you intend to use add_computed_column() instead?'
572
+ )
573
+ return self.add_columns(kwargs, if_exists=if_exists)
553
574
 
554
575
  def add_computed_column(
555
576
  self,
@@ -598,33 +619,35 @@ class Table(SchemaObject):
598
619
 
599
620
  >>> tbl.add_computed_column(rotated=tbl.frame.rotate(90), stored=False)
600
621
  """
601
- self._check_is_dropped()
602
- if self.get_metadata()['is_snapshot']:
603
- raise excs.Error('Cannot add column to a snapshot.')
604
- if len(kwargs) != 1:
605
- raise excs.Error(
606
- f'add_computed_column() requires exactly one keyword argument of the form '
607
- '"column-name=type|value-expression"; '
608
- f'got {len(kwargs)} arguments instead ({", ".join(list(kwargs.keys()))})'
609
- )
610
- col_name, spec = next(iter(kwargs.items()))
611
- if not is_valid_identifier(col_name):
612
- raise excs.Error(f'Invalid column name: {col_name!r}')
613
-
614
- col_schema: dict[str, Any] = {'value': spec}
615
- if stored is not None:
616
- col_schema['stored'] = stored
617
-
618
- # Raise an error if the column expression refers to a column error property
619
- if isinstance(spec, exprs.Expr):
620
- for e in spec.subexprs(expr_class=exprs.ColumnPropertyRef, traverse_matches=False):
621
- if e.is_error_prop():
622
- raise excs.Error(
623
- 'Use of a reference to an error property of another column is not allowed in a computed '
624
- f'column. The specified computation for this column contains this reference: `{e!r}`'
625
- )
622
+ from pixeltable.catalog import Catalog
623
+
624
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
625
+ self._check_is_dropped()
626
+ if self.get_metadata()['is_snapshot']:
627
+ raise excs.Error('Cannot add column to a snapshot.')
628
+ if len(kwargs) != 1:
629
+ raise excs.Error(
630
+ f'add_computed_column() requires exactly one keyword argument of the form '
631
+ '"column-name=type|value-expression"; '
632
+ f'got {len(kwargs)} arguments instead ({", ".join(list(kwargs.keys()))})'
633
+ )
634
+ col_name, spec = next(iter(kwargs.items()))
635
+ if not is_valid_identifier(col_name):
636
+ raise excs.Error(f'Invalid column name: {col_name!r}')
637
+
638
+ col_schema: dict[str, Any] = {'value': spec}
639
+ if stored is not None:
640
+ col_schema['stored'] = stored
641
+
642
+ # Raise an error if the column expression refers to a column error property
643
+ if isinstance(spec, exprs.Expr):
644
+ for e in spec.subexprs(expr_class=exprs.ColumnPropertyRef, traverse_matches=False):
645
+ if e.is_error_prop():
646
+ raise excs.Error(
647
+ 'Use of a reference to an error property of another column is not allowed in a computed '
648
+ f'column. The specified computation for this column contains this reference: `{e!r}`'
649
+ )
626
650
 
627
- with Env.get().begin_xact():
628
651
  # handle existing columns based on if_exists parameter
629
652
  cols_to_ignore = self._ignore_or_drop_existing_columns(
630
653
  [col_name], IfExistsParam.validated(if_exists, 'if_exists')
@@ -781,36 +804,39 @@ class Table(SchemaObject):
781
804
  >>> tbl = pxt.get_table('my_table')
782
805
  ... tbl.drop_col(tbl.col, if_not_exists='ignore')
783
806
  """
784
- self._check_is_dropped()
785
- if self._tbl_version_path.is_snapshot():
786
- raise excs.Error('Cannot drop column from a snapshot.')
787
- col: Column = None
788
- if_not_exists_ = IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
789
- if isinstance(column, str):
790
- col = self._tbl_version_path.get_column(column, include_bases=False)
791
- if col is None:
792
- if if_not_exists_ == IfNotExistsParam.ERROR:
793
- raise excs.Error(f'Column {column!r} unknown')
794
- assert if_not_exists_ == IfNotExistsParam.IGNORE
795
- return
796
- col = self._tbl_version.get().cols_by_name[column]
797
- else:
798
- exists = self._tbl_version_path.has_column(column.col, include_bases=False)
799
- if not exists:
800
- if if_not_exists_ == IfNotExistsParam.ERROR:
801
- raise excs.Error(f'Unknown column: {column.col.qualified_name}')
802
- assert if_not_exists_ == IfNotExistsParam.IGNORE
803
- return
804
- col = column.col
807
+ from pixeltable.catalog import Catalog
805
808
 
806
- dependent_user_cols = [c for c in col.dependent_cols if c.name is not None]
807
- if len(dependent_user_cols) > 0:
808
- raise excs.Error(
809
- f'Cannot drop column `{col.name}` because the following columns depend on it:\n'
810
- f'{", ".join(c.name for c in dependent_user_cols)}'
811
- )
809
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
810
+ self._check_is_dropped()
811
+ if self._tbl_version_path.is_snapshot():
812
+ raise excs.Error('Cannot drop column from a snapshot.')
813
+ col: Column = None
814
+ if_not_exists_ = IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
815
+
816
+ if isinstance(column, str):
817
+ col = self._tbl_version_path.get_column(column, include_bases=False)
818
+ if col is None:
819
+ if if_not_exists_ == IfNotExistsParam.ERROR:
820
+ raise excs.Error(f'Column {column!r} unknown')
821
+ assert if_not_exists_ == IfNotExistsParam.IGNORE
822
+ return
823
+ col = self._tbl_version.get().cols_by_name[column]
824
+ else:
825
+ exists = self._tbl_version_path.has_column(column.col, include_bases=False)
826
+ if not exists:
827
+ if if_not_exists_ == IfNotExistsParam.ERROR:
828
+ raise excs.Error(f'Unknown column: {column.col.qualified_name}')
829
+ assert if_not_exists_ == IfNotExistsParam.IGNORE
830
+ return
831
+ col = column.col
832
+
833
+ dependent_user_cols = [c for c in col.dependent_cols if c.name is not None]
834
+ if len(dependent_user_cols) > 0:
835
+ raise excs.Error(
836
+ f'Cannot drop column `{col.name}` because the following columns depend on it:\n'
837
+ f'{", ".join(c.name for c in dependent_user_cols)}'
838
+ )
812
839
 
813
- with Env.get().begin_xact():
814
840
  # See if this column has a dependent store. We need to look through all stores in all
815
841
  # (transitive) views of this table.
816
842
  dependent_stores = [
@@ -847,7 +873,9 @@ class Table(SchemaObject):
847
873
  >>> tbl = pxt.get_table('my_table')
848
874
  ... tbl.rename_column('col1', 'col2')
849
875
  """
850
- with Env.get().begin_xact():
876
+ from pixeltable.catalog import Catalog
877
+
878
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
851
879
  self._tbl_version.get().rename_column(old_name, new_name)
852
880
 
853
881
  def _list_index_info_for_test(self) -> list[dict[str, Any]]:
@@ -955,11 +983,13 @@ class Table(SchemaObject):
955
983
  ... image_embed=image_embedding_fn
956
984
  ... )
957
985
  """
958
- if self._tbl_version_path.is_snapshot():
959
- raise excs.Error('Cannot add an index to a snapshot')
960
- col = self._resolve_column_parameter(column)
986
+ from pixeltable.catalog import Catalog
987
+
988
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
989
+ if self._tbl_version_path.is_snapshot():
990
+ raise excs.Error('Cannot add an index to a snapshot')
991
+ col = self._resolve_column_parameter(column)
961
992
 
962
- with Env.get().begin_xact():
963
993
  if idx_name is not None and idx_name in self._tbl_version.get().idxs_by_name:
964
994
  if_exists_ = IfExistsParam.validated(if_exists, 'if_exists')
965
995
  # An index with the same name already exists.
@@ -1039,15 +1069,17 @@ class Table(SchemaObject):
1039
1069
  >>> tbl = pxt.get_table('my_table')
1040
1070
  ... tbl.drop_embedding_index(idx_name='idx1', if_not_exists='ignore')
1041
1071
  """
1072
+ from pixeltable.catalog import Catalog
1073
+
1042
1074
  if (column is None) == (idx_name is None):
1043
1075
  raise excs.Error("Exactly one of 'column' or 'idx_name' must be provided")
1044
1076
 
1045
- col: Column = None
1046
- if idx_name is None:
1047
- col = self._resolve_column_parameter(column)
1048
- assert col is not None
1077
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1078
+ col: Column = None
1079
+ if idx_name is None:
1080
+ col = self._resolve_column_parameter(column)
1081
+ assert col is not None
1049
1082
 
1050
- with Env.get().begin_xact():
1051
1083
  self._drop_index(col=col, idx_name=idx_name, _idx_class=index.EmbeddingIndex, if_not_exists=if_not_exists)
1052
1084
 
1053
1085
  def _resolve_column_parameter(self, column: Union[str, ColumnRef]) -> Column:
@@ -1116,15 +1148,17 @@ class Table(SchemaObject):
1116
1148
  ... tbl.drop_index(idx_name='idx1', if_not_exists='ignore')
1117
1149
 
1118
1150
  """
1151
+ from pixeltable.catalog import Catalog
1152
+
1119
1153
  if (column is None) == (idx_name is None):
1120
1154
  raise excs.Error("Exactly one of 'column' or 'idx_name' must be provided")
1121
1155
 
1122
- col: Column = None
1123
- if idx_name is None:
1124
- col = self._resolve_column_parameter(column)
1125
- assert col is not None
1156
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1157
+ col: Column = None
1158
+ if idx_name is None:
1159
+ col = self._resolve_column_parameter(column)
1160
+ assert col is not None
1126
1161
 
1127
- with Env.get().begin_xact():
1128
1162
  self._drop_index(col=col, idx_name=idx_name, if_not_exists=if_not_exists)
1129
1163
 
1130
1164
  def _drop_index(
@@ -1150,7 +1184,7 @@ class Table(SchemaObject):
1150
1184
  else:
1151
1185
  if col.tbl.id != self._tbl_version.id:
1152
1186
  raise excs.Error(
1153
- f'Column {col.name!r}: cannot drop index from column that belongs to base ({col.tbl.get().name}!r)'
1187
+ f'Column {col.name!r}: cannot drop index from column that belongs to base ({col.tbl.name!r})'
1154
1188
  )
1155
1189
  idx_info_list = [info for info in self._tbl_version.get().idxs_by_name.values() if info.col.id == col.id]
1156
1190
  if _idx_class is not None:
@@ -1299,7 +1333,9 @@ class Table(SchemaObject):
1299
1333
 
1300
1334
  >>> tbl.update({'int_col': tbl.int_col + 1}, where=tbl.int_col == 0)
1301
1335
  """
1302
- with Env.get().begin_xact():
1336
+ from pixeltable.catalog import Catalog
1337
+
1338
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1303
1339
  status = self._tbl_version.get().update(value_spec, where, cascade)
1304
1340
  FileCache.get().emit_eviction_warnings()
1305
1341
  return status
@@ -1335,35 +1371,37 @@ class Table(SchemaObject):
1335
1371
  [{'id': 1, 'name': 'Alice', 'age': 30}, {'id': 3, 'name': 'Bob', 'age': 40}],
1336
1372
  if_not_exists='insert')
1337
1373
  """
1338
- if self._tbl_version_path.is_snapshot():
1339
- raise excs.Error('Cannot update a snapshot')
1340
- rows = list(rows)
1374
+ from pixeltable.catalog import Catalog
1341
1375
 
1342
- row_updates: list[dict[Column, exprs.Expr]] = []
1343
- pk_col_names = {c.name for c in self._tbl_version.get().primary_key_columns()}
1376
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1377
+ if self._tbl_version_path.is_snapshot():
1378
+ raise excs.Error('Cannot update a snapshot')
1379
+ rows = list(rows)
1344
1380
 
1345
- # pseudo-column _rowid: contains the rowid of the row to update and can be used instead of the primary key
1346
- has_rowid = _ROWID_COLUMN_NAME in rows[0]
1347
- rowids: list[tuple[int, ...]] = []
1348
- if len(pk_col_names) == 0 and not has_rowid:
1349
- raise excs.Error('Table must have primary key for batch update')
1381
+ row_updates: list[dict[Column, exprs.Expr]] = []
1382
+ pk_col_names = {c.name for c in self._tbl_version.get().primary_key_columns()}
1350
1383
 
1351
- for row_spec in rows:
1352
- col_vals = self._tbl_version.get()._validate_update_spec(
1353
- row_spec, allow_pk=not has_rowid, allow_exprs=False, allow_media=False
1354
- )
1355
- if has_rowid:
1356
- # we expect the _rowid column to be present for each row
1357
- assert _ROWID_COLUMN_NAME in row_spec
1358
- rowids.append(row_spec[_ROWID_COLUMN_NAME])
1359
- else:
1360
- col_names = {col.name for col in col_vals}
1361
- if any(pk_col_name not in col_names for pk_col_name in pk_col_names):
1362
- missing_cols = pk_col_names - {col.name for col in col_vals}
1363
- raise excs.Error(f'Primary key columns ({", ".join(missing_cols)}) missing in {row_spec}')
1364
- row_updates.append(col_vals)
1384
+ # pseudo-column _rowid: contains the rowid of the row to update and can be used instead of the primary key
1385
+ has_rowid = _ROWID_COLUMN_NAME in rows[0]
1386
+ rowids: list[tuple[int, ...]] = []
1387
+ if len(pk_col_names) == 0 and not has_rowid:
1388
+ raise excs.Error('Table must have primary key for batch update')
1389
+
1390
+ for row_spec in rows:
1391
+ col_vals = self._tbl_version.get()._validate_update_spec(
1392
+ row_spec, allow_pk=not has_rowid, allow_exprs=False, allow_media=False
1393
+ )
1394
+ if has_rowid:
1395
+ # we expect the _rowid column to be present for each row
1396
+ assert _ROWID_COLUMN_NAME in row_spec
1397
+ rowids.append(row_spec[_ROWID_COLUMN_NAME])
1398
+ else:
1399
+ col_names = {col.name for col in col_vals}
1400
+ if any(pk_col_name not in col_names for pk_col_name in pk_col_names):
1401
+ missing_cols = pk_col_names - {col.name for col in col_vals}
1402
+ raise excs.Error(f'Primary key columns ({", ".join(missing_cols)}) missing in {row_spec}')
1403
+ row_updates.append(col_vals)
1365
1404
 
1366
- with Env.get().begin_xact():
1367
1405
  status = self._tbl_version.get().batch_update(
1368
1406
  row_updates,
1369
1407
  rowids,
@@ -1397,10 +1435,14 @@ class Table(SchemaObject):
1397
1435
  .. warning::
1398
1436
  This operation is irreversible.
1399
1437
  """
1400
- if self._tbl_version_path.is_snapshot():
1401
- raise excs.Error('Cannot revert a snapshot')
1402
- with Env.get().begin_xact():
1438
+ from pixeltable.catalog import Catalog
1439
+
1440
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1441
+ if self._tbl_version_path.is_snapshot():
1442
+ raise excs.Error('Cannot revert a snapshot')
1403
1443
  self._tbl_version.get().revert()
1444
+ # remove cached md in order to force a reload on the next operation
1445
+ self.__tbl_version_path.clear_cached_md()
1404
1446
 
1405
1447
  @property
1406
1448
  def external_stores(self) -> list[str]:
@@ -1410,12 +1452,16 @@ class Table(SchemaObject):
1410
1452
  """
1411
1453
  Links the specified `ExternalStore` to this table.
1412
1454
  """
1413
- if self._tbl_version.get().is_snapshot:
1414
- raise excs.Error(f'Table `{self._name}` is a snapshot, so it cannot be linked to an external store.')
1415
- if store.name in self.external_stores:
1416
- raise excs.Error(f'Table `{self._name}` already has an external store with that name: {store.name}')
1417
- _logger.info(f'Linking external store `{store.name}` to table `{self._name}`')
1418
- with Env.get().begin_xact():
1455
+ from pixeltable.catalog import Catalog
1456
+
1457
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1458
+ if self._tbl_version.get().is_snapshot:
1459
+ raise excs.Error(f'Table `{self._name}` is a snapshot, so it cannot be linked to an external store.')
1460
+ if store.name in self.external_stores:
1461
+ raise excs.Error(f'Table `{self._name}` already has an external store with that name: {store.name}')
1462
+ _logger.info(f'Linking external store `{store.name}` to table `{self._name}`')
1463
+
1464
+ store.link(self._tbl_version.get()) # might call tbl_version.add_columns()
1419
1465
  self._tbl_version.get().link_external_store(store)
1420
1466
  env.Env.get().console_logger.info(f'Linked external store `{store.name}` to table `{self._name}`.')
1421
1467
 
@@ -1437,24 +1483,32 @@ class Table(SchemaObject):
1437
1483
  delete_external_data (bool): If `True`, then the external data store will also be deleted. WARNING: This
1438
1484
  is a destructive operation that will delete data outside Pixeltable, and cannot be undone.
1439
1485
  """
1440
- self._check_is_dropped()
1441
- all_stores = self.external_stores
1442
-
1443
- if stores is None:
1444
- stores = all_stores
1445
- elif isinstance(stores, str):
1446
- stores = [stores]
1447
-
1448
- # Validation
1449
- if not ignore_errors:
1450
- for store in stores:
1451
- if store not in all_stores:
1452
- raise excs.Error(f'Table `{self._name}` has no external store with that name: {store}')
1453
-
1454
- with Env.get().begin_xact():
1455
- for store in stores:
1456
- self._tbl_version.get().unlink_external_store(store, delete_external_data=delete_external_data)
1457
- env.Env.get().console_logger.info(f'Unlinked external store from table `{self._name}`: {store}')
1486
+ from pixeltable.catalog import Catalog
1487
+
1488
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1489
+ self._check_is_dropped()
1490
+ all_stores = self.external_stores
1491
+
1492
+ if stores is None:
1493
+ stores = all_stores
1494
+ elif isinstance(stores, str):
1495
+ stores = [stores]
1496
+
1497
+ # Validation
1498
+ if not ignore_errors:
1499
+ for store_name in stores:
1500
+ if store_name not in all_stores:
1501
+ raise excs.Error(f'Table `{self._name}` has no external store with that name: {store_name}')
1502
+
1503
+ for store_name in stores:
1504
+ store = self._tbl_version.get().external_stores[store_name]
1505
+ # get hold of the store's debug string before deleting it
1506
+ store_str = str(store)
1507
+ store.unlink(self._tbl_version.get()) # might call tbl_version.drop_columns()
1508
+ self._tbl_version.get().unlink_external_store(store)
1509
+ if delete_external_data and isinstance(store, pxt.io.external_store.Project):
1510
+ store.delete()
1511
+ env.Env.get().console_logger.info(f'Unlinked external store from table `{self._name}`: {store_str}')
1458
1512
 
1459
1513
  def sync(
1460
1514
  self, stores: Optional[str | list[str]] = None, *, export_data: bool = True, import_data: bool = True
@@ -1468,20 +1522,22 @@ class Table(SchemaObject):
1468
1522
  export_data: If `True`, data from this table will be exported to the external stores during synchronization.
1469
1523
  import_data: If `True`, data from the external stores will be imported to this table during synchronization.
1470
1524
  """
1471
- self._check_is_dropped()
1472
- all_stores = self.external_stores
1525
+ from pixeltable.catalog import Catalog
1526
+
1527
+ with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
1528
+ self._check_is_dropped()
1529
+ all_stores = self.external_stores
1473
1530
 
1474
- if stores is None:
1475
- stores = all_stores
1476
- elif isinstance(stores, str):
1477
- stores = [stores]
1531
+ if stores is None:
1532
+ stores = all_stores
1533
+ elif isinstance(stores, str):
1534
+ stores = [stores]
1478
1535
 
1479
- for store in stores:
1480
- if store not in all_stores:
1481
- raise excs.Error(f'Table `{self._name}` has no external store with that name: {store}')
1536
+ for store in stores:
1537
+ if store not in all_stores:
1538
+ raise excs.Error(f'Table `{self._name}` has no external store with that name: {store}')
1482
1539
 
1483
- sync_status = pxt.io.SyncStatus.empty()
1484
- with Env.get().begin_xact():
1540
+ sync_status = pxt.io.SyncStatus.empty()
1485
1541
  for store in stores:
1486
1542
  store_obj = self._tbl_version.get().external_stores[store]
1487
1543
  store_sync_status = store_obj.sync(self, export_data=export_data, import_data=import_data)