pixeltable 0.3.15__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/__version__.py +2 -2
- pixeltable/catalog/catalog.py +292 -105
- pixeltable/catalog/column.py +10 -8
- pixeltable/catalog/dir.py +1 -2
- pixeltable/catalog/insertable_table.py +25 -20
- pixeltable/catalog/schema_object.py +3 -6
- pixeltable/catalog/table.py +245 -189
- pixeltable/catalog/table_version.py +317 -201
- pixeltable/catalog/table_version_handle.py +15 -2
- pixeltable/catalog/table_version_path.py +60 -14
- pixeltable/catalog/view.py +14 -5
- pixeltable/dataframe.py +11 -9
- pixeltable/env.py +2 -4
- pixeltable/exec/in_memory_data_node.py +1 -1
- pixeltable/exec/sql_node.py +20 -11
- pixeltable/exprs/column_property_ref.py +15 -6
- pixeltable/exprs/column_ref.py +32 -11
- pixeltable/exprs/comparison.py +1 -1
- pixeltable/exprs/row_builder.py +4 -6
- pixeltable/exprs/rowid_ref.py +8 -0
- pixeltable/exprs/similarity_expr.py +1 -0
- pixeltable/func/query_template_function.py +1 -1
- pixeltable/functions/string.py +212 -58
- pixeltable/globals.py +7 -4
- pixeltable/index/base.py +5 -0
- pixeltable/index/btree.py +5 -0
- pixeltable/index/embedding_index.py +5 -0
- pixeltable/io/external_store.py +8 -29
- pixeltable/io/label_studio.py +1 -1
- pixeltable/io/parquet.py +2 -2
- pixeltable/io/table_data_conduit.py +0 -31
- pixeltable/metadata/__init__.py +1 -1
- pixeltable/metadata/converters/convert_13.py +2 -2
- pixeltable/metadata/converters/convert_30.py +6 -11
- pixeltable/metadata/converters/convert_35.py +9 -0
- pixeltable/metadata/converters/util.py +3 -9
- pixeltable/metadata/notes.py +1 -0
- pixeltable/metadata/schema.py +5 -1
- pixeltable/plan.py +4 -4
- pixeltable/share/packager.py +24 -9
- pixeltable/share/publish.py +2 -2
- pixeltable/store.py +19 -13
- pixeltable/utils/dbms.py +1 -1
- {pixeltable-0.3.15.dist-info → pixeltable-0.4.0rc1.dist-info}/METADATA +1 -1
- {pixeltable-0.3.15.dist-info → pixeltable-0.4.0rc1.dist-info}/RECORD +48 -47
- {pixeltable-0.3.15.dist-info → pixeltable-0.4.0rc1.dist-info}/LICENSE +0 -0
- {pixeltable-0.3.15.dist-info → pixeltable-0.4.0rc1.dist-info}/WHEEL +0 -0
- {pixeltable-0.3.15.dist-info → pixeltable-0.4.0rc1.dist-info}/entry_points.txt +0 -0
pixeltable/catalog/table.py
CHANGED
|
@@ -4,11 +4,11 @@ import abc
|
|
|
4
4
|
import builtins
|
|
5
5
|
import json
|
|
6
6
|
import logging
|
|
7
|
+
from keyword import iskeyword as is_python_keyword
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import TYPE_CHECKING, Any, Iterable, Literal, Optional, Union, overload
|
|
9
10
|
|
|
10
11
|
from typing import _GenericAlias # type: ignore[attr-defined] # isort: skip
|
|
11
|
-
from keyword import iskeyword as is_python_keyword
|
|
12
12
|
from uuid import UUID
|
|
13
13
|
|
|
14
14
|
import pandas as pd
|
|
@@ -16,7 +16,6 @@ import sqlalchemy as sql
|
|
|
16
16
|
|
|
17
17
|
import pixeltable as pxt
|
|
18
18
|
from pixeltable import catalog, env, exceptions as excs, exprs, index, type_system as ts
|
|
19
|
-
from pixeltable.env import Env
|
|
20
19
|
from pixeltable.metadata import schema
|
|
21
20
|
|
|
22
21
|
from ..exprs import ColumnRef
|
|
@@ -62,11 +61,6 @@ class Table(SchemaObject):
|
|
|
62
61
|
self._is_dropped = False
|
|
63
62
|
self.__tbl_version_path = tbl_version_path
|
|
64
63
|
|
|
65
|
-
# @property
|
|
66
|
-
# def _has_dependents(self) -> bool:
|
|
67
|
-
# """Returns True if this table has any dependent views, or snapshots."""
|
|
68
|
-
# return len(self._get_views(recursive=False)) > 0
|
|
69
|
-
|
|
70
64
|
def _move(self, new_name: str, new_dir_id: UUID) -> None:
|
|
71
65
|
self._check_is_dropped()
|
|
72
66
|
super()._move(new_name, new_dir_id)
|
|
@@ -106,10 +100,12 @@ class Table(SchemaObject):
|
|
|
106
100
|
}
|
|
107
101
|
```
|
|
108
102
|
"""
|
|
109
|
-
|
|
110
|
-
|
|
103
|
+
from pixeltable.catalog import Catalog
|
|
104
|
+
|
|
105
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
106
|
+
self._check_is_dropped()
|
|
111
107
|
md = super().get_metadata()
|
|
112
|
-
md['base'] = self._base_table._path if self._base_table is not None else None
|
|
108
|
+
md['base'] = self._base_table._path() if self._base_table is not None else None
|
|
113
109
|
md['schema'] = self._schema
|
|
114
110
|
md['is_replica'] = self._tbl_version.get().is_replica
|
|
115
111
|
md['version'] = self._version
|
|
@@ -163,9 +159,11 @@ class Table(SchemaObject):
|
|
|
163
159
|
Returns:
|
|
164
160
|
A list of view paths.
|
|
165
161
|
"""
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
162
|
+
from pixeltable.catalog import Catalog
|
|
163
|
+
|
|
164
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
165
|
+
self._check_is_dropped()
|
|
166
|
+
return [t._path() for t in self._get_views(recursive=recursive)]
|
|
169
167
|
|
|
170
168
|
def _get_views(self, *, recursive: bool = True) -> list['Table']:
|
|
171
169
|
cat = catalog.Catalog.get()
|
|
@@ -187,14 +185,20 @@ class Table(SchemaObject):
|
|
|
187
185
|
|
|
188
186
|
See [`DataFrame.select`][pixeltable.DataFrame.select] for more details.
|
|
189
187
|
"""
|
|
190
|
-
|
|
188
|
+
from pixeltable.catalog import Catalog
|
|
189
|
+
|
|
190
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
191
|
+
return self._df().select(*items, **named_items)
|
|
191
192
|
|
|
192
193
|
def where(self, pred: 'exprs.Expr') -> 'pxt.DataFrame':
|
|
193
194
|
"""Filter rows from this table based on the expression.
|
|
194
195
|
|
|
195
196
|
See [`DataFrame.where`][pixeltable.DataFrame.where] for more details.
|
|
196
197
|
"""
|
|
197
|
-
|
|
198
|
+
from pixeltable.catalog import Catalog
|
|
199
|
+
|
|
200
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
201
|
+
return self._df().where(pred)
|
|
198
202
|
|
|
199
203
|
def join(
|
|
200
204
|
self,
|
|
@@ -204,21 +208,30 @@ class Table(SchemaObject):
|
|
|
204
208
|
how: 'pixeltable.plan.JoinType.LiteralType' = 'inner',
|
|
205
209
|
) -> 'pxt.DataFrame':
|
|
206
210
|
"""Join this table with another table."""
|
|
207
|
-
|
|
211
|
+
from pixeltable.catalog import Catalog
|
|
212
|
+
|
|
213
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
214
|
+
return self._df().join(other, on=on, how=how)
|
|
208
215
|
|
|
209
216
|
def order_by(self, *items: 'exprs.Expr', asc: bool = True) -> 'pxt.DataFrame':
|
|
210
217
|
"""Order the rows of this table based on the expression.
|
|
211
218
|
|
|
212
219
|
See [`DataFrame.order_by`][pixeltable.DataFrame.order_by] for more details.
|
|
213
220
|
"""
|
|
214
|
-
|
|
221
|
+
from pixeltable.catalog import Catalog
|
|
222
|
+
|
|
223
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
224
|
+
return self._df().order_by(*items, asc=asc)
|
|
215
225
|
|
|
216
226
|
def group_by(self, *items: 'exprs.Expr') -> 'pxt.DataFrame':
|
|
217
227
|
"""Group the rows of this table based on the expression.
|
|
218
228
|
|
|
219
229
|
See [`DataFrame.group_by`][pixeltable.DataFrame.group_by] for more details.
|
|
220
230
|
"""
|
|
221
|
-
|
|
231
|
+
from pixeltable.catalog import Catalog
|
|
232
|
+
|
|
233
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
234
|
+
return self._df().group_by(*items)
|
|
222
235
|
|
|
223
236
|
def distinct(self) -> 'pxt.DataFrame':
|
|
224
237
|
"""Remove duplicate rows from table."""
|
|
@@ -305,18 +318,21 @@ class Table(SchemaObject):
|
|
|
305
318
|
"""
|
|
306
319
|
Constructs a list of descriptors for this table that can be pretty-printed.
|
|
307
320
|
"""
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
helper.append(
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
321
|
+
from pixeltable.catalog import Catalog
|
|
322
|
+
|
|
323
|
+
with Catalog.get().begin_xact(for_write=False):
|
|
324
|
+
helper = DescriptionHelper()
|
|
325
|
+
helper.append(self._table_descriptor())
|
|
326
|
+
helper.append(self._col_descriptor())
|
|
327
|
+
idxs = self._index_descriptor()
|
|
328
|
+
if not idxs.empty:
|
|
329
|
+
helper.append(idxs)
|
|
330
|
+
stores = self._external_store_descriptor()
|
|
331
|
+
if not stores.empty:
|
|
332
|
+
helper.append(stores)
|
|
333
|
+
if self._comment:
|
|
334
|
+
helper.append(f'COMMENT: {self._comment}')
|
|
335
|
+
return helper
|
|
320
336
|
|
|
321
337
|
def _col_descriptor(self, columns: Optional[list[str]] = None) -> pd.DataFrame:
|
|
322
338
|
return pd.DataFrame(
|
|
@@ -473,15 +489,17 @@ class Table(SchemaObject):
|
|
|
473
489
|
... }
|
|
474
490
|
... tbl.add_columns(schema)
|
|
475
491
|
"""
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
492
|
+
from pixeltable.catalog import Catalog
|
|
493
|
+
|
|
494
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
495
|
+
self._check_is_dropped()
|
|
496
|
+
if self.get_metadata()['is_snapshot']:
|
|
497
|
+
raise excs.Error('Cannot add column to a snapshot.')
|
|
498
|
+
col_schema = {
|
|
499
|
+
col_name: {'type': ts.ColumnType.normalize_type(spec, nullable_default=True, allow_builtin_types=False)}
|
|
500
|
+
for col_name, spec in schema.items()
|
|
501
|
+
}
|
|
502
|
+
|
|
485
503
|
# handle existing columns based on if_exists parameter
|
|
486
504
|
cols_to_ignore = self._ignore_or_drop_existing_columns(
|
|
487
505
|
list(col_schema.keys()), IfExistsParam.validated(if_exists, 'if_exists')
|
|
@@ -534,22 +552,25 @@ class Table(SchemaObject):
|
|
|
534
552
|
|
|
535
553
|
>>> tbl.add_columns({'new_col': pxt.Int})
|
|
536
554
|
"""
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
)
|
|
552
|
-
|
|
555
|
+
from pixeltable.catalog import Catalog
|
|
556
|
+
|
|
557
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
558
|
+
self._check_is_dropped()
|
|
559
|
+
# verify kwargs
|
|
560
|
+
if self._tbl_version.get().is_snapshot:
|
|
561
|
+
raise excs.Error('Cannot add column to a snapshot.')
|
|
562
|
+
# verify kwargs and construct column schema dict
|
|
563
|
+
if len(kwargs) != 1:
|
|
564
|
+
raise excs.Error(
|
|
565
|
+
f'add_column() requires exactly one keyword argument of the form "col_name=col_type"; '
|
|
566
|
+
f'got {len(kwargs)} instead ({", ".join(kwargs.keys())})'
|
|
567
|
+
)
|
|
568
|
+
col_type = next(iter(kwargs.values()))
|
|
569
|
+
if not isinstance(col_type, (ts.ColumnType, type, _GenericAlias)):
|
|
570
|
+
raise excs.Error(
|
|
571
|
+
'The argument to add_column() must be a type; did you intend to use add_computed_column() instead?'
|
|
572
|
+
)
|
|
573
|
+
return self.add_columns(kwargs, if_exists=if_exists)
|
|
553
574
|
|
|
554
575
|
def add_computed_column(
|
|
555
576
|
self,
|
|
@@ -598,33 +619,35 @@ class Table(SchemaObject):
|
|
|
598
619
|
|
|
599
620
|
>>> tbl.add_computed_column(rotated=tbl.frame.rotate(90), stored=False)
|
|
600
621
|
"""
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
622
|
+
from pixeltable.catalog import Catalog
|
|
623
|
+
|
|
624
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
625
|
+
self._check_is_dropped()
|
|
626
|
+
if self.get_metadata()['is_snapshot']:
|
|
627
|
+
raise excs.Error('Cannot add column to a snapshot.')
|
|
628
|
+
if len(kwargs) != 1:
|
|
629
|
+
raise excs.Error(
|
|
630
|
+
f'add_computed_column() requires exactly one keyword argument of the form '
|
|
631
|
+
'"column-name=type|value-expression"; '
|
|
632
|
+
f'got {len(kwargs)} arguments instead ({", ".join(list(kwargs.keys()))})'
|
|
633
|
+
)
|
|
634
|
+
col_name, spec = next(iter(kwargs.items()))
|
|
635
|
+
if not is_valid_identifier(col_name):
|
|
636
|
+
raise excs.Error(f'Invalid column name: {col_name!r}')
|
|
637
|
+
|
|
638
|
+
col_schema: dict[str, Any] = {'value': spec}
|
|
639
|
+
if stored is not None:
|
|
640
|
+
col_schema['stored'] = stored
|
|
641
|
+
|
|
642
|
+
# Raise an error if the column expression refers to a column error property
|
|
643
|
+
if isinstance(spec, exprs.Expr):
|
|
644
|
+
for e in spec.subexprs(expr_class=exprs.ColumnPropertyRef, traverse_matches=False):
|
|
645
|
+
if e.is_error_prop():
|
|
646
|
+
raise excs.Error(
|
|
647
|
+
'Use of a reference to an error property of another column is not allowed in a computed '
|
|
648
|
+
f'column. The specified computation for this column contains this reference: `{e!r}`'
|
|
649
|
+
)
|
|
626
650
|
|
|
627
|
-
with Env.get().begin_xact():
|
|
628
651
|
# handle existing columns based on if_exists parameter
|
|
629
652
|
cols_to_ignore = self._ignore_or_drop_existing_columns(
|
|
630
653
|
[col_name], IfExistsParam.validated(if_exists, 'if_exists')
|
|
@@ -781,36 +804,39 @@ class Table(SchemaObject):
|
|
|
781
804
|
>>> tbl = pxt.get_table('my_table')
|
|
782
805
|
... tbl.drop_col(tbl.col, if_not_exists='ignore')
|
|
783
806
|
"""
|
|
784
|
-
|
|
785
|
-
if self._tbl_version_path.is_snapshot():
|
|
786
|
-
raise excs.Error('Cannot drop column from a snapshot.')
|
|
787
|
-
col: Column = None
|
|
788
|
-
if_not_exists_ = IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
|
|
789
|
-
if isinstance(column, str):
|
|
790
|
-
col = self._tbl_version_path.get_column(column, include_bases=False)
|
|
791
|
-
if col is None:
|
|
792
|
-
if if_not_exists_ == IfNotExistsParam.ERROR:
|
|
793
|
-
raise excs.Error(f'Column {column!r} unknown')
|
|
794
|
-
assert if_not_exists_ == IfNotExistsParam.IGNORE
|
|
795
|
-
return
|
|
796
|
-
col = self._tbl_version.get().cols_by_name[column]
|
|
797
|
-
else:
|
|
798
|
-
exists = self._tbl_version_path.has_column(column.col, include_bases=False)
|
|
799
|
-
if not exists:
|
|
800
|
-
if if_not_exists_ == IfNotExistsParam.ERROR:
|
|
801
|
-
raise excs.Error(f'Unknown column: {column.col.qualified_name}')
|
|
802
|
-
assert if_not_exists_ == IfNotExistsParam.IGNORE
|
|
803
|
-
return
|
|
804
|
-
col = column.col
|
|
807
|
+
from pixeltable.catalog import Catalog
|
|
805
808
|
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
)
|
|
809
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
810
|
+
self._check_is_dropped()
|
|
811
|
+
if self._tbl_version_path.is_snapshot():
|
|
812
|
+
raise excs.Error('Cannot drop column from a snapshot.')
|
|
813
|
+
col: Column = None
|
|
814
|
+
if_not_exists_ = IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
|
|
815
|
+
|
|
816
|
+
if isinstance(column, str):
|
|
817
|
+
col = self._tbl_version_path.get_column(column, include_bases=False)
|
|
818
|
+
if col is None:
|
|
819
|
+
if if_not_exists_ == IfNotExistsParam.ERROR:
|
|
820
|
+
raise excs.Error(f'Column {column!r} unknown')
|
|
821
|
+
assert if_not_exists_ == IfNotExistsParam.IGNORE
|
|
822
|
+
return
|
|
823
|
+
col = self._tbl_version.get().cols_by_name[column]
|
|
824
|
+
else:
|
|
825
|
+
exists = self._tbl_version_path.has_column(column.col, include_bases=False)
|
|
826
|
+
if not exists:
|
|
827
|
+
if if_not_exists_ == IfNotExistsParam.ERROR:
|
|
828
|
+
raise excs.Error(f'Unknown column: {column.col.qualified_name}')
|
|
829
|
+
assert if_not_exists_ == IfNotExistsParam.IGNORE
|
|
830
|
+
return
|
|
831
|
+
col = column.col
|
|
832
|
+
|
|
833
|
+
dependent_user_cols = [c for c in col.dependent_cols if c.name is not None]
|
|
834
|
+
if len(dependent_user_cols) > 0:
|
|
835
|
+
raise excs.Error(
|
|
836
|
+
f'Cannot drop column `{col.name}` because the following columns depend on it:\n'
|
|
837
|
+
f'{", ".join(c.name for c in dependent_user_cols)}'
|
|
838
|
+
)
|
|
812
839
|
|
|
813
|
-
with Env.get().begin_xact():
|
|
814
840
|
# See if this column has a dependent store. We need to look through all stores in all
|
|
815
841
|
# (transitive) views of this table.
|
|
816
842
|
dependent_stores = [
|
|
@@ -847,7 +873,9 @@ class Table(SchemaObject):
|
|
|
847
873
|
>>> tbl = pxt.get_table('my_table')
|
|
848
874
|
... tbl.rename_column('col1', 'col2')
|
|
849
875
|
"""
|
|
850
|
-
|
|
876
|
+
from pixeltable.catalog import Catalog
|
|
877
|
+
|
|
878
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
851
879
|
self._tbl_version.get().rename_column(old_name, new_name)
|
|
852
880
|
|
|
853
881
|
def _list_index_info_for_test(self) -> list[dict[str, Any]]:
|
|
@@ -955,11 +983,13 @@ class Table(SchemaObject):
|
|
|
955
983
|
... image_embed=image_embedding_fn
|
|
956
984
|
... )
|
|
957
985
|
"""
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
986
|
+
from pixeltable.catalog import Catalog
|
|
987
|
+
|
|
988
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
989
|
+
if self._tbl_version_path.is_snapshot():
|
|
990
|
+
raise excs.Error('Cannot add an index to a snapshot')
|
|
991
|
+
col = self._resolve_column_parameter(column)
|
|
961
992
|
|
|
962
|
-
with Env.get().begin_xact():
|
|
963
993
|
if idx_name is not None and idx_name in self._tbl_version.get().idxs_by_name:
|
|
964
994
|
if_exists_ = IfExistsParam.validated(if_exists, 'if_exists')
|
|
965
995
|
# An index with the same name already exists.
|
|
@@ -1039,15 +1069,17 @@ class Table(SchemaObject):
|
|
|
1039
1069
|
>>> tbl = pxt.get_table('my_table')
|
|
1040
1070
|
... tbl.drop_embedding_index(idx_name='idx1', if_not_exists='ignore')
|
|
1041
1071
|
"""
|
|
1072
|
+
from pixeltable.catalog import Catalog
|
|
1073
|
+
|
|
1042
1074
|
if (column is None) == (idx_name is None):
|
|
1043
1075
|
raise excs.Error("Exactly one of 'column' or 'idx_name' must be provided")
|
|
1044
1076
|
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1077
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1078
|
+
col: Column = None
|
|
1079
|
+
if idx_name is None:
|
|
1080
|
+
col = self._resolve_column_parameter(column)
|
|
1081
|
+
assert col is not None
|
|
1049
1082
|
|
|
1050
|
-
with Env.get().begin_xact():
|
|
1051
1083
|
self._drop_index(col=col, idx_name=idx_name, _idx_class=index.EmbeddingIndex, if_not_exists=if_not_exists)
|
|
1052
1084
|
|
|
1053
1085
|
def _resolve_column_parameter(self, column: Union[str, ColumnRef]) -> Column:
|
|
@@ -1116,15 +1148,17 @@ class Table(SchemaObject):
|
|
|
1116
1148
|
... tbl.drop_index(idx_name='idx1', if_not_exists='ignore')
|
|
1117
1149
|
|
|
1118
1150
|
"""
|
|
1151
|
+
from pixeltable.catalog import Catalog
|
|
1152
|
+
|
|
1119
1153
|
if (column is None) == (idx_name is None):
|
|
1120
1154
|
raise excs.Error("Exactly one of 'column' or 'idx_name' must be provided")
|
|
1121
1155
|
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1156
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1157
|
+
col: Column = None
|
|
1158
|
+
if idx_name is None:
|
|
1159
|
+
col = self._resolve_column_parameter(column)
|
|
1160
|
+
assert col is not None
|
|
1126
1161
|
|
|
1127
|
-
with Env.get().begin_xact():
|
|
1128
1162
|
self._drop_index(col=col, idx_name=idx_name, if_not_exists=if_not_exists)
|
|
1129
1163
|
|
|
1130
1164
|
def _drop_index(
|
|
@@ -1150,7 +1184,7 @@ class Table(SchemaObject):
|
|
|
1150
1184
|
else:
|
|
1151
1185
|
if col.tbl.id != self._tbl_version.id:
|
|
1152
1186
|
raise excs.Error(
|
|
1153
|
-
f'Column {col.name!r}: cannot drop index from column that belongs to base ({col.tbl.
|
|
1187
|
+
f'Column {col.name!r}: cannot drop index from column that belongs to base ({col.tbl.name!r})'
|
|
1154
1188
|
)
|
|
1155
1189
|
idx_info_list = [info for info in self._tbl_version.get().idxs_by_name.values() if info.col.id == col.id]
|
|
1156
1190
|
if _idx_class is not None:
|
|
@@ -1299,7 +1333,9 @@ class Table(SchemaObject):
|
|
|
1299
1333
|
|
|
1300
1334
|
>>> tbl.update({'int_col': tbl.int_col + 1}, where=tbl.int_col == 0)
|
|
1301
1335
|
"""
|
|
1302
|
-
|
|
1336
|
+
from pixeltable.catalog import Catalog
|
|
1337
|
+
|
|
1338
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1303
1339
|
status = self._tbl_version.get().update(value_spec, where, cascade)
|
|
1304
1340
|
FileCache.get().emit_eviction_warnings()
|
|
1305
1341
|
return status
|
|
@@ -1335,35 +1371,37 @@ class Table(SchemaObject):
|
|
|
1335
1371
|
[{'id': 1, 'name': 'Alice', 'age': 30}, {'id': 3, 'name': 'Bob', 'age': 40}],
|
|
1336
1372
|
if_not_exists='insert')
|
|
1337
1373
|
"""
|
|
1338
|
-
|
|
1339
|
-
raise excs.Error('Cannot update a snapshot')
|
|
1340
|
-
rows = list(rows)
|
|
1374
|
+
from pixeltable.catalog import Catalog
|
|
1341
1375
|
|
|
1342
|
-
|
|
1343
|
-
|
|
1376
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1377
|
+
if self._tbl_version_path.is_snapshot():
|
|
1378
|
+
raise excs.Error('Cannot update a snapshot')
|
|
1379
|
+
rows = list(rows)
|
|
1344
1380
|
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
rowids: list[tuple[int, ...]] = []
|
|
1348
|
-
if len(pk_col_names) == 0 and not has_rowid:
|
|
1349
|
-
raise excs.Error('Table must have primary key for batch update')
|
|
1381
|
+
row_updates: list[dict[Column, exprs.Expr]] = []
|
|
1382
|
+
pk_col_names = {c.name for c in self._tbl_version.get().primary_key_columns()}
|
|
1350
1383
|
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
)
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
if
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1384
|
+
# pseudo-column _rowid: contains the rowid of the row to update and can be used instead of the primary key
|
|
1385
|
+
has_rowid = _ROWID_COLUMN_NAME in rows[0]
|
|
1386
|
+
rowids: list[tuple[int, ...]] = []
|
|
1387
|
+
if len(pk_col_names) == 0 and not has_rowid:
|
|
1388
|
+
raise excs.Error('Table must have primary key for batch update')
|
|
1389
|
+
|
|
1390
|
+
for row_spec in rows:
|
|
1391
|
+
col_vals = self._tbl_version.get()._validate_update_spec(
|
|
1392
|
+
row_spec, allow_pk=not has_rowid, allow_exprs=False, allow_media=False
|
|
1393
|
+
)
|
|
1394
|
+
if has_rowid:
|
|
1395
|
+
# we expect the _rowid column to be present for each row
|
|
1396
|
+
assert _ROWID_COLUMN_NAME in row_spec
|
|
1397
|
+
rowids.append(row_spec[_ROWID_COLUMN_NAME])
|
|
1398
|
+
else:
|
|
1399
|
+
col_names = {col.name for col in col_vals}
|
|
1400
|
+
if any(pk_col_name not in col_names for pk_col_name in pk_col_names):
|
|
1401
|
+
missing_cols = pk_col_names - {col.name for col in col_vals}
|
|
1402
|
+
raise excs.Error(f'Primary key columns ({", ".join(missing_cols)}) missing in {row_spec}')
|
|
1403
|
+
row_updates.append(col_vals)
|
|
1365
1404
|
|
|
1366
|
-
with Env.get().begin_xact():
|
|
1367
1405
|
status = self._tbl_version.get().batch_update(
|
|
1368
1406
|
row_updates,
|
|
1369
1407
|
rowids,
|
|
@@ -1397,10 +1435,14 @@ class Table(SchemaObject):
|
|
|
1397
1435
|
.. warning::
|
|
1398
1436
|
This operation is irreversible.
|
|
1399
1437
|
"""
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
with
|
|
1438
|
+
from pixeltable.catalog import Catalog
|
|
1439
|
+
|
|
1440
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1441
|
+
if self._tbl_version_path.is_snapshot():
|
|
1442
|
+
raise excs.Error('Cannot revert a snapshot')
|
|
1403
1443
|
self._tbl_version.get().revert()
|
|
1444
|
+
# remove cached md in order to force a reload on the next operation
|
|
1445
|
+
self.__tbl_version_path.clear_cached_md()
|
|
1404
1446
|
|
|
1405
1447
|
@property
|
|
1406
1448
|
def external_stores(self) -> list[str]:
|
|
@@ -1410,12 +1452,16 @@ class Table(SchemaObject):
|
|
|
1410
1452
|
"""
|
|
1411
1453
|
Links the specified `ExternalStore` to this table.
|
|
1412
1454
|
"""
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1455
|
+
from pixeltable.catalog import Catalog
|
|
1456
|
+
|
|
1457
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1458
|
+
if self._tbl_version.get().is_snapshot:
|
|
1459
|
+
raise excs.Error(f'Table `{self._name}` is a snapshot, so it cannot be linked to an external store.')
|
|
1460
|
+
if store.name in self.external_stores:
|
|
1461
|
+
raise excs.Error(f'Table `{self._name}` already has an external store with that name: {store.name}')
|
|
1462
|
+
_logger.info(f'Linking external store `{store.name}` to table `{self._name}`')
|
|
1463
|
+
|
|
1464
|
+
store.link(self._tbl_version.get()) # might call tbl_version.add_columns()
|
|
1419
1465
|
self._tbl_version.get().link_external_store(store)
|
|
1420
1466
|
env.Env.get().console_logger.info(f'Linked external store `{store.name}` to table `{self._name}`.')
|
|
1421
1467
|
|
|
@@ -1437,24 +1483,32 @@ class Table(SchemaObject):
|
|
|
1437
1483
|
delete_external_data (bool): If `True`, then the external data store will also be deleted. WARNING: This
|
|
1438
1484
|
is a destructive operation that will delete data outside Pixeltable, and cannot be undone.
|
|
1439
1485
|
"""
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
stores
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1486
|
+
from pixeltable.catalog import Catalog
|
|
1487
|
+
|
|
1488
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1489
|
+
self._check_is_dropped()
|
|
1490
|
+
all_stores = self.external_stores
|
|
1491
|
+
|
|
1492
|
+
if stores is None:
|
|
1493
|
+
stores = all_stores
|
|
1494
|
+
elif isinstance(stores, str):
|
|
1495
|
+
stores = [stores]
|
|
1496
|
+
|
|
1497
|
+
# Validation
|
|
1498
|
+
if not ignore_errors:
|
|
1499
|
+
for store_name in stores:
|
|
1500
|
+
if store_name not in all_stores:
|
|
1501
|
+
raise excs.Error(f'Table `{self._name}` has no external store with that name: {store_name}')
|
|
1502
|
+
|
|
1503
|
+
for store_name in stores:
|
|
1504
|
+
store = self._tbl_version.get().external_stores[store_name]
|
|
1505
|
+
# get hold of the store's debug string before deleting it
|
|
1506
|
+
store_str = str(store)
|
|
1507
|
+
store.unlink(self._tbl_version.get()) # might call tbl_version.drop_columns()
|
|
1508
|
+
self._tbl_version.get().unlink_external_store(store)
|
|
1509
|
+
if delete_external_data and isinstance(store, pxt.io.external_store.Project):
|
|
1510
|
+
store.delete()
|
|
1511
|
+
env.Env.get().console_logger.info(f'Unlinked external store from table `{self._name}`: {store_str}')
|
|
1458
1512
|
|
|
1459
1513
|
def sync(
|
|
1460
1514
|
self, stores: Optional[str | list[str]] = None, *, export_data: bool = True, import_data: bool = True
|
|
@@ -1468,20 +1522,22 @@ class Table(SchemaObject):
|
|
|
1468
1522
|
export_data: If `True`, data from this table will be exported to the external stores during synchronization.
|
|
1469
1523
|
import_data: If `True`, data from the external stores will be imported to this table during synchronization.
|
|
1470
1524
|
"""
|
|
1471
|
-
|
|
1472
|
-
|
|
1525
|
+
from pixeltable.catalog import Catalog
|
|
1526
|
+
|
|
1527
|
+
with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
|
|
1528
|
+
self._check_is_dropped()
|
|
1529
|
+
all_stores = self.external_stores
|
|
1473
1530
|
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1531
|
+
if stores is None:
|
|
1532
|
+
stores = all_stores
|
|
1533
|
+
elif isinstance(stores, str):
|
|
1534
|
+
stores = [stores]
|
|
1478
1535
|
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1536
|
+
for store in stores:
|
|
1537
|
+
if store not in all_stores:
|
|
1538
|
+
raise excs.Error(f'Table `{self._name}` has no external store with that name: {store}')
|
|
1482
1539
|
|
|
1483
|
-
|
|
1484
|
-
with Env.get().begin_xact():
|
|
1540
|
+
sync_status = pxt.io.SyncStatus.empty()
|
|
1485
1541
|
for store in stores:
|
|
1486
1542
|
store_obj = self._tbl_version.get().external_stores[store]
|
|
1487
1543
|
store_sync_status = store_obj.sync(self, export_data=export_data, import_data=import_data)
|