pixeltable 0.3.7__py3-none-any.whl → 0.3.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/__version__.py +2 -2
- pixeltable/catalog/catalog.py +509 -103
- pixeltable/catalog/column.py +1 -0
- pixeltable/catalog/dir.py +15 -6
- pixeltable/catalog/path.py +15 -0
- pixeltable/catalog/schema_object.py +7 -12
- pixeltable/catalog/table.py +3 -12
- pixeltable/catalog/table_version.py +5 -0
- pixeltable/catalog/view.py +0 -4
- pixeltable/env.py +14 -8
- pixeltable/exprs/__init__.py +2 -0
- pixeltable/exprs/arithmetic_expr.py +7 -11
- pixeltable/exprs/array_slice.py +1 -1
- pixeltable/exprs/column_property_ref.py +3 -3
- pixeltable/exprs/column_ref.py +5 -6
- pixeltable/exprs/comparison.py +2 -5
- pixeltable/exprs/compound_predicate.py +4 -4
- pixeltable/exprs/expr.py +32 -19
- pixeltable/exprs/expr_dict.py +3 -3
- pixeltable/exprs/expr_set.py +1 -1
- pixeltable/exprs/function_call.py +28 -41
- pixeltable/exprs/globals.py +3 -3
- pixeltable/exprs/in_predicate.py +1 -1
- pixeltable/exprs/inline_expr.py +3 -3
- pixeltable/exprs/is_null.py +1 -1
- pixeltable/exprs/json_mapper.py +5 -5
- pixeltable/exprs/json_path.py +27 -15
- pixeltable/exprs/literal.py +1 -1
- pixeltable/exprs/method_ref.py +2 -2
- pixeltable/exprs/row_builder.py +3 -5
- pixeltable/exprs/rowid_ref.py +4 -7
- pixeltable/exprs/similarity_expr.py +5 -5
- pixeltable/exprs/sql_element_cache.py +1 -1
- pixeltable/exprs/type_cast.py +2 -3
- pixeltable/exprs/variable.py +2 -2
- pixeltable/ext/__init__.py +2 -0
- pixeltable/ext/functions/__init__.py +2 -0
- pixeltable/ext/functions/yolox.py +3 -3
- pixeltable/func/__init__.py +2 -0
- pixeltable/func/aggregate_function.py +9 -9
- pixeltable/func/callable_function.py +7 -5
- pixeltable/func/expr_template_function.py +6 -16
- pixeltable/func/function.py +10 -8
- pixeltable/func/function_registry.py +1 -3
- pixeltable/func/query_template_function.py +8 -24
- pixeltable/func/signature.py +23 -22
- pixeltable/func/tools.py +3 -3
- pixeltable/func/udf.py +5 -3
- pixeltable/globals.py +118 -260
- pixeltable/share/__init__.py +2 -0
- pixeltable/share/packager.py +3 -3
- pixeltable/share/publish.py +3 -5
- pixeltable/utils/coco.py +4 -4
- pixeltable/utils/console_output.py +1 -3
- pixeltable/utils/coroutine.py +41 -0
- pixeltable/utils/description_helper.py +1 -1
- pixeltable/utils/documents.py +3 -3
- pixeltable/utils/filecache.py +18 -8
- pixeltable/utils/formatter.py +2 -3
- pixeltable/utils/media_store.py +1 -1
- pixeltable/utils/pytorch.py +1 -1
- pixeltable/utils/sql.py +4 -4
- pixeltable/utils/transactional_directory.py +2 -1
- {pixeltable-0.3.7.dist-info → pixeltable-0.3.9.dist-info}/METADATA +1 -1
- {pixeltable-0.3.7.dist-info → pixeltable-0.3.9.dist-info}/RECORD +68 -67
- {pixeltable-0.3.7.dist-info → pixeltable-0.3.9.dist-info}/LICENSE +0 -0
- {pixeltable-0.3.7.dist-info → pixeltable-0.3.9.dist-info}/WHEEL +0 -0
- {pixeltable-0.3.7.dist-info → pixeltable-0.3.9.dist-info}/entry_points.txt +0 -0
pixeltable/catalog/column.py
CHANGED
pixeltable/catalog/dir.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import dataclasses
|
|
4
|
+
import datetime
|
|
5
|
+
import json
|
|
4
6
|
import logging
|
|
5
7
|
from uuid import UUID
|
|
6
8
|
|
|
7
9
|
import sqlalchemy as sql
|
|
10
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
|
8
11
|
|
|
9
12
|
from pixeltable.env import Env
|
|
10
13
|
from pixeltable.metadata import schema
|
|
@@ -26,6 +29,7 @@ class Dir(SchemaObject):
|
|
|
26
29
|
dir_record = schema.Dir(parent_id=parent_id, md=dataclasses.asdict(dir_md))
|
|
27
30
|
session.add(dir_record)
|
|
28
31
|
session.flush()
|
|
32
|
+
# print(f'{datetime.datetime.now()} create dir {dir_record}')
|
|
29
33
|
assert dir_record.id is not None
|
|
30
34
|
assert isinstance(dir_record.id, UUID)
|
|
31
35
|
dir = cls(dir_record.id, parent_id, name)
|
|
@@ -43,11 +47,16 @@ class Dir(SchemaObject):
|
|
|
43
47
|
return super()._path()
|
|
44
48
|
|
|
45
49
|
def _move(self, new_name: str, new_dir_id: UUID) -> None:
|
|
50
|
+
# print(
|
|
51
|
+
# f'{datetime.datetime.now()} move dir name={self._name} parent={self._dir_id} new_name={new_name} new_dir_id={new_dir_id}'
|
|
52
|
+
# )
|
|
46
53
|
super()._move(new_name, new_dir_id)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
54
|
+
stmt = sql.text(
|
|
55
|
+
(
|
|
56
|
+
f'UPDATE {schema.Dir.__table__} '
|
|
57
|
+
f'SET {schema.Dir.parent_id.name} = :new_dir_id, '
|
|
58
|
+
f" {schema.Dir.md.name}['name'] = :new_name "
|
|
59
|
+
f'WHERE {schema.Dir.id.name} = :id'
|
|
53
60
|
)
|
|
61
|
+
)
|
|
62
|
+
Env.get().conn.execute(stmt, {'new_dir_id': new_dir_id, 'new_name': json.dumps(new_name), 'id': self._id})
|
pixeltable/catalog/path.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
from typing import Iterator
|
|
4
5
|
|
|
5
6
|
from pixeltable import exceptions as excs
|
|
6
7
|
|
|
@@ -55,5 +56,19 @@ class Path:
|
|
|
55
56
|
is_prefix = self.components == other.components[: self.len]
|
|
56
57
|
return is_prefix and (self.len == (other.len - 1) or not is_parent)
|
|
57
58
|
|
|
59
|
+
def ancestors(self) -> Iterator[Path]:
|
|
60
|
+
"""
|
|
61
|
+
Return all ancestors of this path in top-down order including root.
|
|
62
|
+
If this path is for the root directory, which has no parent, then None is returned.
|
|
63
|
+
"""
|
|
64
|
+
if self.is_root:
|
|
65
|
+
return
|
|
66
|
+
else:
|
|
67
|
+
for i in range(0, len(self.components)):
|
|
68
|
+
yield Path('.'.join(self.components[0:i]), empty_is_valid=True)
|
|
69
|
+
|
|
58
70
|
def __str__(self) -> str:
|
|
59
71
|
return '.'.join(self.components)
|
|
72
|
+
|
|
73
|
+
def __lt__(self, other: Path) -> bool:
|
|
74
|
+
return str(self) < str(other)
|
|
@@ -2,7 +2,7 @@ from abc import abstractmethod
|
|
|
2
2
|
from typing import TYPE_CHECKING, Any, Optional
|
|
3
3
|
from uuid import UUID
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
from pixeltable.env import Env
|
|
6
6
|
|
|
7
7
|
if TYPE_CHECKING:
|
|
8
8
|
from pixeltable import catalog
|
|
@@ -28,24 +28,19 @@ class SchemaObject:
|
|
|
28
28
|
"""Returns the parent directory of this schema object."""
|
|
29
29
|
from .catalog import Catalog
|
|
30
30
|
|
|
31
|
-
with
|
|
31
|
+
with Env.get().begin_xact():
|
|
32
32
|
if self._dir_id is None:
|
|
33
33
|
return None
|
|
34
34
|
return Catalog.get().get_dir(self._dir_id)
|
|
35
35
|
|
|
36
36
|
def _path(self) -> str:
|
|
37
37
|
"""Returns the path to this schema object."""
|
|
38
|
-
|
|
39
|
-
from .catalog import Catalog
|
|
38
|
+
from .catalog import Catalog
|
|
40
39
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
# root directory. Either way, we return just the name.
|
|
46
|
-
return self._name
|
|
47
|
-
else:
|
|
48
|
-
return f'{dir_path}.{self._name}'
|
|
40
|
+
assert self._dir_id is not None
|
|
41
|
+
with Env.get().begin_xact():
|
|
42
|
+
path = Catalog.get().get_dir_path(self._dir_id)
|
|
43
|
+
return str(path.append(self._name))
|
|
49
44
|
|
|
50
45
|
def get_metadata(self) -> dict[str, Any]:
|
|
51
46
|
"""Returns metadata associated with this schema object."""
|
pixeltable/catalog/table.py
CHANGED
|
@@ -171,8 +171,8 @@ class Table(SchemaObject):
|
|
|
171
171
|
|
|
172
172
|
def _get_views(self, *, recursive: bool = True) -> list['Table']:
|
|
173
173
|
cat = catalog.Catalog.get()
|
|
174
|
-
view_ids = cat.
|
|
175
|
-
views = [cat.
|
|
174
|
+
view_ids = cat.get_view_ids(self._id)
|
|
175
|
+
views = [cat.get_table_by_id(id) for id in view_ids]
|
|
176
176
|
if recursive:
|
|
177
177
|
views.extend([t for view in views for t in view._get_views(recursive=True)])
|
|
178
178
|
return views
|
|
@@ -265,7 +265,7 @@ class Table(SchemaObject):
|
|
|
265
265
|
if self._tbl_version_path.base is None:
|
|
266
266
|
return None
|
|
267
267
|
base_id = self._tbl_version_path.base.tbl_version.id
|
|
268
|
-
return catalog.Catalog.get().
|
|
268
|
+
return catalog.Catalog.get().get_table_by_id(base_id)
|
|
269
269
|
|
|
270
270
|
@property
|
|
271
271
|
def _bases(self) -> list['Table']:
|
|
@@ -369,11 +369,6 @@ class Table(SchemaObject):
|
|
|
369
369
|
pd_rows.append(row)
|
|
370
370
|
return pd.DataFrame(pd_rows)
|
|
371
371
|
|
|
372
|
-
def ensure_md_loaded(self) -> None:
|
|
373
|
-
"""Ensure that table metadata is loaded."""
|
|
374
|
-
for col in self._tbl_version.get().cols_by_id.values():
|
|
375
|
-
_ = col.value_expr
|
|
376
|
-
|
|
377
372
|
def describe(self) -> None:
|
|
378
373
|
"""
|
|
379
374
|
Print the table schema.
|
|
@@ -387,13 +382,9 @@ class Table(SchemaObject):
|
|
|
387
382
|
print(repr(self))
|
|
388
383
|
|
|
389
384
|
def _drop(self) -> None:
|
|
390
|
-
cat = catalog.Catalog.get()
|
|
391
385
|
self._check_is_dropped()
|
|
392
386
|
self._tbl_version.get().drop()
|
|
393
387
|
self._is_dropped = True
|
|
394
|
-
# update catalog
|
|
395
|
-
cat = catalog.Catalog.get()
|
|
396
|
-
cat.remove_tbl(self._id)
|
|
397
388
|
|
|
398
389
|
# TODO Factor this out into a separate module.
|
|
399
390
|
# The return type is unresolvable, but torch can't be imported since it's an optional dependency.
|
|
@@ -454,6 +454,11 @@ class TableVersion:
|
|
|
454
454
|
)
|
|
455
455
|
)
|
|
456
456
|
|
|
457
|
+
def ensure_md_loaded(self) -> None:
|
|
458
|
+
"""Ensure that table metadata is loaded."""
|
|
459
|
+
for col in self.cols_by_id.values():
|
|
460
|
+
_ = col.value_expr
|
|
461
|
+
|
|
457
462
|
def _store_idx_name(self, idx_id: int) -> str:
|
|
458
463
|
"""Return name of index in the store, which needs to be globally unique"""
|
|
459
464
|
return f'idx_{self.id.hex}_{idx_id}'
|
pixeltable/catalog/view.py
CHANGED
|
@@ -237,15 +237,11 @@ class View(Table):
|
|
|
237
237
|
)
|
|
238
238
|
|
|
239
239
|
def _drop(self) -> None:
|
|
240
|
-
cat = catalog.Catalog.get()
|
|
241
240
|
if self._snapshot_only:
|
|
242
241
|
# there is not TableVersion to drop
|
|
243
242
|
self._check_is_dropped()
|
|
244
243
|
self.is_dropped = True
|
|
245
244
|
TableVersion.delete_md(self._id)
|
|
246
|
-
# update catalog
|
|
247
|
-
cat = catalog.Catalog.get()
|
|
248
|
-
cat.remove_tbl(self._id)
|
|
249
245
|
else:
|
|
250
246
|
super()._drop()
|
|
251
247
|
|
pixeltable/env.py
CHANGED
|
@@ -170,19 +170,25 @@ class Env:
|
|
|
170
170
|
assert self._current_session is not None
|
|
171
171
|
return self._current_session
|
|
172
172
|
|
|
173
|
+
def in_xact(self) -> bool:
|
|
174
|
+
return self._current_conn is not None
|
|
175
|
+
|
|
173
176
|
@contextmanager
|
|
174
177
|
def begin_xact(self) -> Iterator[sql.Connection]:
|
|
175
178
|
"""Return a context manager that yields a connection to the database. Idempotent."""
|
|
176
179
|
if self._current_conn is None:
|
|
177
180
|
assert self._current_session is None
|
|
178
|
-
|
|
179
|
-
self.
|
|
180
|
-
|
|
181
|
-
|
|
181
|
+
try:
|
|
182
|
+
with self.engine.begin() as conn, sql.orm.Session(conn) as session:
|
|
183
|
+
# TODO: remove print() once we're done with debugging the concurrent update behavior
|
|
184
|
+
# print(f'{datetime.datetime.now()}: start xact')
|
|
185
|
+
self._current_conn = conn
|
|
186
|
+
self._current_session = session
|
|
182
187
|
yield conn
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
188
|
+
finally:
|
|
189
|
+
self._current_session = None
|
|
190
|
+
self._current_conn = None
|
|
191
|
+
# print(f'{datetime.datetime.now()}: end xact')
|
|
186
192
|
else:
|
|
187
193
|
assert self._current_session is not None
|
|
188
194
|
yield self._current_conn
|
|
@@ -391,7 +397,7 @@ class Env:
|
|
|
391
397
|
def _create_engine(self, time_zone_name: Optional[str], echo: bool = False) -> None:
|
|
392
398
|
connect_args = {} if time_zone_name is None else {'options': f'-c timezone={time_zone_name}'}
|
|
393
399
|
self._sa_engine = sql.create_engine(
|
|
394
|
-
self.db_url, echo=echo,
|
|
400
|
+
self.db_url, echo=echo, isolation_level='REPEATABLE READ', connect_args=connect_args
|
|
395
401
|
)
|
|
396
402
|
self._logger.info(f'Created SQLAlchemy engine at: {self.db_url}')
|
|
397
403
|
with self.engine.begin() as conn:
|
pixeltable/exprs/__init__.py
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Any, Optional
|
|
3
|
+
from typing import Any, Optional
|
|
4
4
|
|
|
5
5
|
import sqlalchemy as sql
|
|
6
6
|
|
|
7
|
-
import
|
|
8
|
-
import pixeltable.exprs as exprs
|
|
9
|
-
import pixeltable.type_system as ts
|
|
7
|
+
from pixeltable import exceptions as excs, type_system as ts
|
|
10
8
|
|
|
11
9
|
from .data_row import DataRow
|
|
12
10
|
from .expr import Expr
|
|
@@ -50,13 +48,13 @@ class ArithmeticExpr(Expr):
|
|
|
50
48
|
# add parentheses around operands that are ArithmeticExprs to express precedence
|
|
51
49
|
op1_str = f'({self._op1})' if isinstance(self._op1, ArithmeticExpr) else str(self._op1)
|
|
52
50
|
op2_str = f'({self._op2})' if isinstance(self._op2, ArithmeticExpr) else str(self._op2)
|
|
53
|
-
return f'{op1_str} {
|
|
51
|
+
return f'{op1_str} {self.operator} {op2_str}'
|
|
54
52
|
|
|
55
53
|
def _equals(self, other: ArithmeticExpr) -> bool:
|
|
56
54
|
return self.operator == other.operator
|
|
57
55
|
|
|
58
56
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
59
|
-
return super()._id_attrs()
|
|
57
|
+
return [*super()._id_attrs(), ('operator', self.operator.value)]
|
|
60
58
|
|
|
61
59
|
def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
|
|
62
60
|
assert self.col_type.is_int_type() or self.col_type.is_float_type() or self.col_type.is_json_type()
|
|
@@ -95,7 +93,7 @@ class ArithmeticExpr(Expr):
|
|
|
95
93
|
return sql.sql.expression.cast(sql.func.floor(left / nullif), self.col_type.to_sa_type())
|
|
96
94
|
if self.col_type.is_float_type():
|
|
97
95
|
return sql.sql.expression.cast(sql.func.floor(left / nullif), self.col_type.to_sa_type())
|
|
98
|
-
|
|
96
|
+
raise AssertionError()
|
|
99
97
|
|
|
100
98
|
def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
|
|
101
99
|
op1_val = data_row[self._op1.slot_idx]
|
|
@@ -113,9 +111,7 @@ class ArithmeticExpr(Expr):
|
|
|
113
111
|
|
|
114
112
|
data_row[self.slot_idx] = self.eval_nullable(op1_val, op2_val)
|
|
115
113
|
|
|
116
|
-
def eval_nullable(
|
|
117
|
-
self, op1_val: Union[int, float, None], op2_val: Union[int, float, None]
|
|
118
|
-
) -> Union[int, float, None]:
|
|
114
|
+
def eval_nullable(self, op1_val: Optional[float], op2_val: Optional[float]) -> Optional[float]:
|
|
119
115
|
"""
|
|
120
116
|
Return the result of evaluating the expression on two nullable int/float operands,
|
|
121
117
|
None is interpreted as SQL NULL
|
|
@@ -124,7 +120,7 @@ class ArithmeticExpr(Expr):
|
|
|
124
120
|
return None
|
|
125
121
|
return self.eval_non_null(op1_val, op2_val)
|
|
126
122
|
|
|
127
|
-
def eval_non_null(self, op1_val:
|
|
123
|
+
def eval_non_null(self, op1_val: float, op2_val: float) -> float:
|
|
128
124
|
"""
|
|
129
125
|
Return the result of evaluating the expression on two int/float operands
|
|
130
126
|
"""
|
pixeltable/exprs/array_slice.py
CHANGED
|
@@ -41,7 +41,7 @@ class ArraySlice(Expr):
|
|
|
41
41
|
return self.index == other.index
|
|
42
42
|
|
|
43
43
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
44
|
-
return super()._id_attrs()
|
|
44
|
+
return [*super()._id_attrs(), ('index', self.index)]
|
|
45
45
|
|
|
46
46
|
def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
|
|
47
47
|
return None
|
|
@@ -40,7 +40,7 @@ class ColumnPropertyRef(Expr):
|
|
|
40
40
|
return self.prop == other.prop
|
|
41
41
|
|
|
42
42
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
43
|
-
return super()._id_attrs()
|
|
43
|
+
return [*super()._id_attrs(), ('prop', self.prop.value)]
|
|
44
44
|
|
|
45
45
|
@property
|
|
46
46
|
def _col_ref(self) -> ColumnRef:
|
|
@@ -52,7 +52,7 @@ class ColumnPropertyRef(Expr):
|
|
|
52
52
|
return f'{self._col_ref}.{self.prop.name.lower()}'
|
|
53
53
|
|
|
54
54
|
def is_error_prop(self) -> bool:
|
|
55
|
-
return self.prop
|
|
55
|
+
return self.prop in {self.Property.ERRORTYPE, self.Property.ERRORMSG}
|
|
56
56
|
|
|
57
57
|
def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
|
|
58
58
|
if not self._col_ref.col.is_stored:
|
|
@@ -95,7 +95,7 @@ class ColumnPropertyRef(Expr):
|
|
|
95
95
|
else:
|
|
96
96
|
data_row[self.slot_idx] = str(exc)
|
|
97
97
|
else:
|
|
98
|
-
|
|
98
|
+
raise AssertionError()
|
|
99
99
|
|
|
100
100
|
def _as_dict(self) -> dict:
|
|
101
101
|
return {'prop': self.prop.value, **super()._as_dict()}
|
pixeltable/exprs/column_ref.py
CHANGED
|
@@ -6,9 +6,7 @@ from uuid import UUID
|
|
|
6
6
|
import sqlalchemy as sql
|
|
7
7
|
|
|
8
8
|
import pixeltable as pxt
|
|
9
|
-
import
|
|
10
|
-
import pixeltable.exceptions as excs
|
|
11
|
-
import pixeltable.iterators as iters
|
|
9
|
+
from pixeltable import catalog, exceptions as excs, iterators as iters
|
|
12
10
|
|
|
13
11
|
from ..utils.description_helper import DescriptionHelper
|
|
14
12
|
from .data_row import DataRow
|
|
@@ -84,7 +82,8 @@ class ColumnRef(Expr):
|
|
|
84
82
|
assert len(self.iter_arg_ctx.target_slot_idxs) == 1 # a single inline dict
|
|
85
83
|
|
|
86
84
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
87
|
-
return
|
|
85
|
+
return [
|
|
86
|
+
*super()._id_attrs(),
|
|
88
87
|
('tbl_id', self.col.tbl.id),
|
|
89
88
|
('col_id', self.col.id),
|
|
90
89
|
('perform_validation', self.perform_validation),
|
|
@@ -138,7 +137,7 @@ class ColumnRef(Expr):
|
|
|
138
137
|
return self.col == other.col and self.perform_validation == other.perform_validation
|
|
139
138
|
|
|
140
139
|
def _df(self) -> 'pxt.dataframe.DataFrame':
|
|
141
|
-
tbl = catalog.Catalog.get().
|
|
140
|
+
tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
|
|
142
141
|
return tbl.select(self)
|
|
143
142
|
|
|
144
143
|
def show(self, *args, **kwargs) -> 'pxt.dataframe.DataFrameResultSet':
|
|
@@ -166,7 +165,7 @@ class ColumnRef(Expr):
|
|
|
166
165
|
return self._descriptors().to_html()
|
|
167
166
|
|
|
168
167
|
def _descriptors(self) -> DescriptionHelper:
|
|
169
|
-
tbl = catalog.Catalog.get().
|
|
168
|
+
tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
|
|
170
169
|
helper = DescriptionHelper()
|
|
171
170
|
helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path()!r})')
|
|
172
171
|
helper.append(tbl._col_descriptor([self.col.name]))
|
pixeltable/exprs/comparison.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import
|
|
3
|
+
from typing import Any, Optional
|
|
4
4
|
|
|
5
5
|
import sqlalchemy as sql
|
|
6
6
|
|
|
@@ -15,9 +15,6 @@ from .literal import Literal
|
|
|
15
15
|
from .row_builder import RowBuilder
|
|
16
16
|
from .sql_element_cache import SqlElementCache
|
|
17
17
|
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from pixeltable import index
|
|
20
|
-
|
|
21
18
|
|
|
22
19
|
class Comparison(Expr):
|
|
23
20
|
is_search_arg_comparison: bool
|
|
@@ -62,7 +59,7 @@ class Comparison(Expr):
|
|
|
62
59
|
return self.operator == other.operator
|
|
63
60
|
|
|
64
61
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
65
|
-
return super()._id_attrs()
|
|
62
|
+
return [*super()._id_attrs(), ('operator', self.operator.value)]
|
|
66
63
|
|
|
67
64
|
@property
|
|
68
65
|
def _op1(self) -> Expr:
|
|
@@ -5,7 +5,7 @@ from typing import Any, Callable, Optional
|
|
|
5
5
|
|
|
6
6
|
import sqlalchemy as sql
|
|
7
7
|
|
|
8
|
-
import
|
|
8
|
+
from pixeltable import type_system as ts
|
|
9
9
|
|
|
10
10
|
from .data_row import DataRow
|
|
11
11
|
from .expr import Expr
|
|
@@ -58,10 +58,10 @@ class CompoundPredicate(Expr):
|
|
|
58
58
|
return self.operator == other.operator
|
|
59
59
|
|
|
60
60
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
61
|
-
return super()._id_attrs()
|
|
61
|
+
return [*super()._id_attrs(), ('operator', self.operator.value)]
|
|
62
62
|
|
|
63
63
|
def split_conjuncts(self, condition: Callable[[Expr], bool]) -> tuple[list[Expr], Optional[Expr]]:
|
|
64
|
-
if self.operator
|
|
64
|
+
if self.operator in {LogicalOperator.OR, LogicalOperator.NOT}:
|
|
65
65
|
return super().split_conjuncts(condition)
|
|
66
66
|
matches = [op for op in self.components if condition(op)]
|
|
67
67
|
non_matches = [op for op in self.components if not condition(op)]
|
|
@@ -83,7 +83,7 @@ class CompoundPredicate(Expr):
|
|
|
83
83
|
if self.operator == LogicalOperator.NOT:
|
|
84
84
|
data_row[self.slot_idx] = not data_row[self.components[0].slot_idx]
|
|
85
85
|
else:
|
|
86
|
-
val =
|
|
86
|
+
val = self.operator == LogicalOperator.AND
|
|
87
87
|
op_function = operator.and_ if self.operator == LogicalOperator.AND else operator.or_
|
|
88
88
|
for op in self.components:
|
|
89
89
|
val = op_function(val, data_row[op.slot_idx])
|
pixeltable/exprs/expr.py
CHANGED
|
@@ -90,14 +90,29 @@ class Expr(abc.ABC):
|
|
|
90
90
|
result = c_scope
|
|
91
91
|
return result
|
|
92
92
|
|
|
93
|
-
def bind_rel_paths(self
|
|
93
|
+
def bind_rel_paths(self) -> None:
|
|
94
94
|
"""
|
|
95
95
|
Binds relative JsonPaths to mapper.
|
|
96
96
|
This needs to be done in a separate phase after __init__(), because RelativeJsonPath()(-1) cannot be resolved
|
|
97
97
|
by the immediately containing JsonMapper during initialization.
|
|
98
98
|
"""
|
|
99
|
+
self._bind_rel_paths()
|
|
100
|
+
assert not self._has_relative_path, self._expr_tree()
|
|
101
|
+
|
|
102
|
+
def _bind_rel_paths(self, mapper: Optional['exprs.JsonMapper'] = None) -> None:
|
|
103
|
+
for c in self.components:
|
|
104
|
+
c._bind_rel_paths(mapper)
|
|
105
|
+
|
|
106
|
+
def _expr_tree(self) -> str:
|
|
107
|
+
"""Returns a string representation of this expression as a multi-line tree. Useful for debugging."""
|
|
108
|
+
buf: list[str] = []
|
|
109
|
+
self._expr_tree_r(0, buf)
|
|
110
|
+
return '\n'.join(buf)
|
|
111
|
+
|
|
112
|
+
def _expr_tree_r(self, indent: int, buf: list[str]) -> None:
|
|
113
|
+
buf.append(f'{" " * indent}{type(self).__name__}: {self}'.replace('\n', '\\n'))
|
|
99
114
|
for c in self.components:
|
|
100
|
-
c.
|
|
115
|
+
c._expr_tree_r(indent + 2, buf)
|
|
101
116
|
|
|
102
117
|
def default_column_name(self) -> Optional[str]:
|
|
103
118
|
"""
|
|
@@ -129,7 +144,7 @@ class Expr(abc.ABC):
|
|
|
129
144
|
"""
|
|
130
145
|
Subclass-specific comparison. Implemented as a function because __eq__() is needed to construct Comparisons.
|
|
131
146
|
"""
|
|
132
|
-
if type(self)
|
|
147
|
+
if type(self) is not type(other):
|
|
133
148
|
return False
|
|
134
149
|
if len(self.components) != len(other.components):
|
|
135
150
|
return False
|
|
@@ -171,10 +186,7 @@ class Expr(abc.ABC):
|
|
|
171
186
|
def list_equals(cls, a: list[Expr], b: list[Expr]) -> bool:
|
|
172
187
|
if len(a) != len(b):
|
|
173
188
|
return False
|
|
174
|
-
for i in range(len(a))
|
|
175
|
-
if not a[i].equals(b[i]):
|
|
176
|
-
return False
|
|
177
|
-
return True
|
|
189
|
+
return all(a[i].equals(b[i]) for i in range(len(a)))
|
|
178
190
|
|
|
179
191
|
def copy(self) -> Expr:
|
|
180
192
|
"""
|
|
@@ -216,9 +228,9 @@ class Expr(abc.ABC):
|
|
|
216
228
|
return new.copy()
|
|
217
229
|
for i in range(len(self.components)):
|
|
218
230
|
self.components[i] = self.components[i].substitute(spec)
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
return
|
|
231
|
+
result = self.maybe_literal()
|
|
232
|
+
result.id = result._create_id()
|
|
233
|
+
return result
|
|
222
234
|
|
|
223
235
|
@classmethod
|
|
224
236
|
def list_substitute(cls, expr_list: list[Expr], spec: dict[Expr, Expr]) -> None:
|
|
@@ -253,10 +265,7 @@ class Expr(abc.ABC):
|
|
|
253
265
|
from .column_ref import ColumnRef
|
|
254
266
|
|
|
255
267
|
col_refs = self.subexprs(ColumnRef)
|
|
256
|
-
for col_ref in col_refs
|
|
257
|
-
if not any(tbl.has_column(col_ref.col) for tbl in tbls):
|
|
258
|
-
return False
|
|
259
|
-
return True
|
|
268
|
+
return all(any(tbl.has_column(col_ref.col) for tbl in tbls) for col_ref in col_refs)
|
|
260
269
|
|
|
261
270
|
def retarget(self, tbl: catalog.TableVersionPath) -> Self:
|
|
262
271
|
"""Retarget ColumnRefs in this expr to the specific TableVersions in tbl."""
|
|
@@ -361,6 +370,10 @@ class Expr(abc.ABC):
|
|
|
361
370
|
except StopIteration:
|
|
362
371
|
return False
|
|
363
372
|
|
|
373
|
+
@property
|
|
374
|
+
def _has_relative_path(self) -> bool:
|
|
375
|
+
return any(c._has_relative_path for c in self.components)
|
|
376
|
+
|
|
364
377
|
def tbl_ids(self) -> set[UUID]:
|
|
365
378
|
"""Returns table ids referenced by this expr."""
|
|
366
379
|
from .column_ref import ColumnRef
|
|
@@ -370,7 +383,7 @@ class Expr(abc.ABC):
|
|
|
370
383
|
|
|
371
384
|
@classmethod
|
|
372
385
|
def all_tbl_ids(cls, exprs_: Iterable[Expr]) -> set[UUID]:
|
|
373
|
-
return
|
|
386
|
+
return {tbl_id for e in exprs_ for tbl_id in e.tbl_ids()}
|
|
374
387
|
|
|
375
388
|
@classmethod
|
|
376
389
|
def get_refd_columns(cls, expr_dict: dict[str, Any]) -> list[catalog.Column]:
|
|
@@ -489,7 +502,7 @@ class Expr(abc.ABC):
|
|
|
489
502
|
return {'_classname': self.__class__.__name__, **self._as_dict()}
|
|
490
503
|
|
|
491
504
|
@classmethod
|
|
492
|
-
def as_dict_list(
|
|
505
|
+
def as_dict_list(cls, expr_list: list[Expr]) -> list[dict]:
|
|
493
506
|
return [e.as_dict() for e in expr_list]
|
|
494
507
|
|
|
495
508
|
def _as_dict(self) -> dict:
|
|
@@ -520,7 +533,7 @@ class Expr(abc.ABC):
|
|
|
520
533
|
|
|
521
534
|
@classmethod
|
|
522
535
|
def _from_dict(cls, d: dict, components: list[Expr]) -> Self:
|
|
523
|
-
|
|
536
|
+
raise AssertionError(f'not implemented: {cls.__name__}')
|
|
524
537
|
|
|
525
538
|
def isin(self, value_set: Any) -> 'exprs.InPredicate':
|
|
526
539
|
from .in_predicate import InPredicate
|
|
@@ -792,13 +805,13 @@ class Expr(abc.ABC):
|
|
|
792
805
|
first_param = next(params_iter) if len(params) >= 1 else None
|
|
793
806
|
second_param = next(params_iter) if len(params) >= 2 else None
|
|
794
807
|
# Check that fn has at least one positional parameter
|
|
795
|
-
if len(params) == 0 or first_param.kind in
|
|
808
|
+
if len(params) == 0 or first_param.kind in {inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.VAR_KEYWORD}:
|
|
796
809
|
raise excs.Error(f'Function `{fn.__name__}` has no positional parameters.')
|
|
797
810
|
# Check that fn has at most one required parameter, i.e., its second parameter
|
|
798
811
|
# has no default and is not a varargs
|
|
799
812
|
if (
|
|
800
813
|
len(params) >= 2
|
|
801
|
-
and second_param.kind not in
|
|
814
|
+
and second_param.kind not in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD}
|
|
802
815
|
and second_param.default is inspect.Parameter.empty
|
|
803
816
|
):
|
|
804
817
|
raise excs.Error(f'Function `{fn.__name__}` has multiple required parameters.')
|
pixeltable/exprs/expr_dict.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from typing import Generic, Iterable, Iterator, Optional, TypeVar
|
|
2
2
|
|
|
3
|
-
T = TypeVar('T')
|
|
4
|
-
|
|
5
3
|
from .expr import Expr
|
|
6
4
|
|
|
5
|
+
T = TypeVar('T')
|
|
6
|
+
|
|
7
7
|
|
|
8
8
|
class ExprDict(Generic[T]):
|
|
9
9
|
"""
|
|
@@ -47,7 +47,7 @@ class ExprDict(Generic[T]):
|
|
|
47
47
|
self._data.clear()
|
|
48
48
|
|
|
49
49
|
def keys(self) -> Iterator[Expr]:
|
|
50
|
-
return self
|
|
50
|
+
return iter(self)
|
|
51
51
|
|
|
52
52
|
def values(self) -> Iterator[T]:
|
|
53
53
|
return (value for _, value in self._data.values())
|
pixeltable/exprs/expr_set.py
CHANGED
|
@@ -46,7 +46,7 @@ class ExprSet(Generic[T]):
|
|
|
46
46
|
|
|
47
47
|
def __getitem__(self, index: object) -> Optional[T]:
|
|
48
48
|
"""Indexed lookup by slot_idx or Expr.id."""
|
|
49
|
-
assert isinstance(index, int
|
|
49
|
+
assert isinstance(index, (int, Expr))
|
|
50
50
|
if isinstance(index, int):
|
|
51
51
|
# return expr with matching slot_idx
|
|
52
52
|
return self.exprs_by_idx.get(index)
|