pixeltable 0.3.14__py3-none-any.whl → 0.5.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. pixeltable/__init__.py +42 -8
  2. pixeltable/{dataframe.py → _query.py} +470 -206
  3. pixeltable/_version.py +1 -0
  4. pixeltable/catalog/__init__.py +5 -4
  5. pixeltable/catalog/catalog.py +1785 -432
  6. pixeltable/catalog/column.py +190 -113
  7. pixeltable/catalog/dir.py +2 -4
  8. pixeltable/catalog/globals.py +19 -46
  9. pixeltable/catalog/insertable_table.py +191 -98
  10. pixeltable/catalog/path.py +63 -23
  11. pixeltable/catalog/schema_object.py +11 -15
  12. pixeltable/catalog/table.py +843 -436
  13. pixeltable/catalog/table_metadata.py +103 -0
  14. pixeltable/catalog/table_version.py +978 -657
  15. pixeltable/catalog/table_version_handle.py +72 -16
  16. pixeltable/catalog/table_version_path.py +112 -43
  17. pixeltable/catalog/tbl_ops.py +53 -0
  18. pixeltable/catalog/update_status.py +191 -0
  19. pixeltable/catalog/view.py +134 -90
  20. pixeltable/config.py +134 -22
  21. pixeltable/env.py +471 -157
  22. pixeltable/exceptions.py +6 -0
  23. pixeltable/exec/__init__.py +4 -1
  24. pixeltable/exec/aggregation_node.py +7 -8
  25. pixeltable/exec/cache_prefetch_node.py +83 -110
  26. pixeltable/exec/cell_materialization_node.py +268 -0
  27. pixeltable/exec/cell_reconstruction_node.py +168 -0
  28. pixeltable/exec/component_iteration_node.py +4 -3
  29. pixeltable/exec/data_row_batch.py +8 -65
  30. pixeltable/exec/exec_context.py +16 -4
  31. pixeltable/exec/exec_node.py +13 -36
  32. pixeltable/exec/expr_eval/evaluators.py +11 -7
  33. pixeltable/exec/expr_eval/expr_eval_node.py +27 -12
  34. pixeltable/exec/expr_eval/globals.py +8 -5
  35. pixeltable/exec/expr_eval/row_buffer.py +1 -2
  36. pixeltable/exec/expr_eval/schedulers.py +106 -56
  37. pixeltable/exec/globals.py +35 -0
  38. pixeltable/exec/in_memory_data_node.py +19 -19
  39. pixeltable/exec/object_store_save_node.py +293 -0
  40. pixeltable/exec/row_update_node.py +16 -9
  41. pixeltable/exec/sql_node.py +351 -84
  42. pixeltable/exprs/__init__.py +1 -1
  43. pixeltable/exprs/arithmetic_expr.py +27 -22
  44. pixeltable/exprs/array_slice.py +3 -3
  45. pixeltable/exprs/column_property_ref.py +36 -23
  46. pixeltable/exprs/column_ref.py +213 -89
  47. pixeltable/exprs/comparison.py +5 -5
  48. pixeltable/exprs/compound_predicate.py +5 -4
  49. pixeltable/exprs/data_row.py +164 -54
  50. pixeltable/exprs/expr.py +70 -44
  51. pixeltable/exprs/expr_dict.py +3 -3
  52. pixeltable/exprs/expr_set.py +17 -10
  53. pixeltable/exprs/function_call.py +100 -40
  54. pixeltable/exprs/globals.py +2 -2
  55. pixeltable/exprs/in_predicate.py +4 -4
  56. pixeltable/exprs/inline_expr.py +18 -32
  57. pixeltable/exprs/is_null.py +7 -3
  58. pixeltable/exprs/json_mapper.py +8 -8
  59. pixeltable/exprs/json_path.py +56 -22
  60. pixeltable/exprs/literal.py +27 -5
  61. pixeltable/exprs/method_ref.py +2 -2
  62. pixeltable/exprs/object_ref.py +2 -2
  63. pixeltable/exprs/row_builder.py +167 -67
  64. pixeltable/exprs/rowid_ref.py +25 -10
  65. pixeltable/exprs/similarity_expr.py +58 -40
  66. pixeltable/exprs/sql_element_cache.py +4 -4
  67. pixeltable/exprs/string_op.py +5 -5
  68. pixeltable/exprs/type_cast.py +3 -5
  69. pixeltable/func/__init__.py +1 -0
  70. pixeltable/func/aggregate_function.py +8 -8
  71. pixeltable/func/callable_function.py +9 -9
  72. pixeltable/func/expr_template_function.py +17 -11
  73. pixeltable/func/function.py +18 -20
  74. pixeltable/func/function_registry.py +6 -7
  75. pixeltable/func/globals.py +2 -3
  76. pixeltable/func/mcp.py +74 -0
  77. pixeltable/func/query_template_function.py +29 -27
  78. pixeltable/func/signature.py +46 -19
  79. pixeltable/func/tools.py +31 -13
  80. pixeltable/func/udf.py +18 -20
  81. pixeltable/functions/__init__.py +16 -0
  82. pixeltable/functions/anthropic.py +123 -77
  83. pixeltable/functions/audio.py +147 -10
  84. pixeltable/functions/bedrock.py +13 -6
  85. pixeltable/functions/date.py +7 -4
  86. pixeltable/functions/deepseek.py +35 -43
  87. pixeltable/functions/document.py +81 -0
  88. pixeltable/functions/fal.py +76 -0
  89. pixeltable/functions/fireworks.py +11 -20
  90. pixeltable/functions/gemini.py +195 -39
  91. pixeltable/functions/globals.py +142 -14
  92. pixeltable/functions/groq.py +108 -0
  93. pixeltable/functions/huggingface.py +1056 -24
  94. pixeltable/functions/image.py +115 -57
  95. pixeltable/functions/json.py +1 -1
  96. pixeltable/functions/llama_cpp.py +28 -13
  97. pixeltable/functions/math.py +67 -5
  98. pixeltable/functions/mistralai.py +18 -55
  99. pixeltable/functions/net.py +70 -0
  100. pixeltable/functions/ollama.py +20 -13
  101. pixeltable/functions/openai.py +240 -226
  102. pixeltable/functions/openrouter.py +143 -0
  103. pixeltable/functions/replicate.py +4 -4
  104. pixeltable/functions/reve.py +250 -0
  105. pixeltable/functions/string.py +239 -69
  106. pixeltable/functions/timestamp.py +16 -16
  107. pixeltable/functions/together.py +24 -84
  108. pixeltable/functions/twelvelabs.py +188 -0
  109. pixeltable/functions/util.py +6 -1
  110. pixeltable/functions/uuid.py +30 -0
  111. pixeltable/functions/video.py +1515 -107
  112. pixeltable/functions/vision.py +8 -8
  113. pixeltable/functions/voyageai.py +289 -0
  114. pixeltable/functions/whisper.py +16 -8
  115. pixeltable/functions/whisperx.py +179 -0
  116. pixeltable/{ext/functions → functions}/yolox.py +2 -4
  117. pixeltable/globals.py +362 -115
  118. pixeltable/index/base.py +17 -21
  119. pixeltable/index/btree.py +28 -22
  120. pixeltable/index/embedding_index.py +100 -118
  121. pixeltable/io/__init__.py +4 -2
  122. pixeltable/io/datarows.py +8 -7
  123. pixeltable/io/external_store.py +56 -105
  124. pixeltable/io/fiftyone.py +13 -13
  125. pixeltable/io/globals.py +31 -30
  126. pixeltable/io/hf_datasets.py +61 -16
  127. pixeltable/io/label_studio.py +74 -70
  128. pixeltable/io/lancedb.py +3 -0
  129. pixeltable/io/pandas.py +21 -12
  130. pixeltable/io/parquet.py +25 -105
  131. pixeltable/io/table_data_conduit.py +250 -123
  132. pixeltable/io/utils.py +4 -4
  133. pixeltable/iterators/__init__.py +2 -1
  134. pixeltable/iterators/audio.py +26 -25
  135. pixeltable/iterators/base.py +9 -3
  136. pixeltable/iterators/document.py +112 -78
  137. pixeltable/iterators/image.py +12 -15
  138. pixeltable/iterators/string.py +11 -4
  139. pixeltable/iterators/video.py +523 -120
  140. pixeltable/metadata/__init__.py +14 -3
  141. pixeltable/metadata/converters/convert_13.py +2 -2
  142. pixeltable/metadata/converters/convert_18.py +2 -2
  143. pixeltable/metadata/converters/convert_19.py +2 -2
  144. pixeltable/metadata/converters/convert_20.py +2 -2
  145. pixeltable/metadata/converters/convert_21.py +2 -2
  146. pixeltable/metadata/converters/convert_22.py +2 -2
  147. pixeltable/metadata/converters/convert_24.py +2 -2
  148. pixeltable/metadata/converters/convert_25.py +2 -2
  149. pixeltable/metadata/converters/convert_26.py +2 -2
  150. pixeltable/metadata/converters/convert_29.py +4 -4
  151. pixeltable/metadata/converters/convert_30.py +34 -21
  152. pixeltable/metadata/converters/convert_34.py +2 -2
  153. pixeltable/metadata/converters/convert_35.py +9 -0
  154. pixeltable/metadata/converters/convert_36.py +38 -0
  155. pixeltable/metadata/converters/convert_37.py +15 -0
  156. pixeltable/metadata/converters/convert_38.py +39 -0
  157. pixeltable/metadata/converters/convert_39.py +124 -0
  158. pixeltable/metadata/converters/convert_40.py +73 -0
  159. pixeltable/metadata/converters/convert_41.py +12 -0
  160. pixeltable/metadata/converters/convert_42.py +9 -0
  161. pixeltable/metadata/converters/convert_43.py +44 -0
  162. pixeltable/metadata/converters/util.py +20 -31
  163. pixeltable/metadata/notes.py +9 -0
  164. pixeltable/metadata/schema.py +140 -53
  165. pixeltable/metadata/utils.py +74 -0
  166. pixeltable/mypy/__init__.py +3 -0
  167. pixeltable/mypy/mypy_plugin.py +123 -0
  168. pixeltable/plan.py +382 -115
  169. pixeltable/share/__init__.py +1 -1
  170. pixeltable/share/packager.py +547 -83
  171. pixeltable/share/protocol/__init__.py +33 -0
  172. pixeltable/share/protocol/common.py +165 -0
  173. pixeltable/share/protocol/operation_types.py +33 -0
  174. pixeltable/share/protocol/replica.py +119 -0
  175. pixeltable/share/publish.py +257 -59
  176. pixeltable/store.py +311 -194
  177. pixeltable/type_system.py +373 -211
  178. pixeltable/utils/__init__.py +2 -3
  179. pixeltable/utils/arrow.py +131 -17
  180. pixeltable/utils/av.py +298 -0
  181. pixeltable/utils/azure_store.py +346 -0
  182. pixeltable/utils/coco.py +6 -6
  183. pixeltable/utils/code.py +3 -3
  184. pixeltable/utils/console_output.py +4 -1
  185. pixeltable/utils/coroutine.py +6 -23
  186. pixeltable/utils/dbms.py +32 -6
  187. pixeltable/utils/description_helper.py +4 -5
  188. pixeltable/utils/documents.py +7 -18
  189. pixeltable/utils/exception_handler.py +7 -30
  190. pixeltable/utils/filecache.py +6 -6
  191. pixeltable/utils/formatter.py +86 -48
  192. pixeltable/utils/gcs_store.py +295 -0
  193. pixeltable/utils/http.py +133 -0
  194. pixeltable/utils/http_server.py +2 -3
  195. pixeltable/utils/iceberg.py +1 -2
  196. pixeltable/utils/image.py +17 -0
  197. pixeltable/utils/lancedb.py +90 -0
  198. pixeltable/utils/local_store.py +322 -0
  199. pixeltable/utils/misc.py +5 -0
  200. pixeltable/utils/object_stores.py +573 -0
  201. pixeltable/utils/pydantic.py +60 -0
  202. pixeltable/utils/pytorch.py +5 -6
  203. pixeltable/utils/s3_store.py +527 -0
  204. pixeltable/utils/sql.py +26 -0
  205. pixeltable/utils/system.py +30 -0
  206. pixeltable-0.5.7.dist-info/METADATA +579 -0
  207. pixeltable-0.5.7.dist-info/RECORD +227 -0
  208. {pixeltable-0.3.14.dist-info → pixeltable-0.5.7.dist-info}/WHEEL +1 -1
  209. pixeltable-0.5.7.dist-info/entry_points.txt +2 -0
  210. pixeltable/__version__.py +0 -3
  211. pixeltable/catalog/named_function.py +0 -40
  212. pixeltable/ext/__init__.py +0 -17
  213. pixeltable/ext/functions/__init__.py +0 -11
  214. pixeltable/ext/functions/whisperx.py +0 -77
  215. pixeltable/utils/media_store.py +0 -77
  216. pixeltable/utils/s3.py +0 -17
  217. pixeltable-0.3.14.dist-info/METADATA +0 -434
  218. pixeltable-0.3.14.dist-info/RECORD +0 -186
  219. pixeltable-0.3.14.dist-info/entry_points.txt +0 -3
  220. {pixeltable-0.3.14.dist-info → pixeltable-0.5.7.dist-info/licenses}/LICENSE +0 -0
@@ -1,4 +1,6 @@
1
- from typing import Any, Optional
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any
2
4
 
3
5
  import sqlalchemy as sql
4
6
 
@@ -12,35 +14,38 @@ from .literal import Literal
12
14
  from .row_builder import RowBuilder
13
15
  from .sql_element_cache import SqlElementCache
14
16
 
17
+ if TYPE_CHECKING:
18
+ from pixeltable.catalog.table_version import TableVersion
19
+
15
20
 
16
21
  class SimilarityExpr(Expr):
17
- def __init__(self, col_ref: ColumnRef, item: Any, idx_name: Optional[str] = None):
18
- super().__init__(ts.FloatType())
19
- item_expr = Expr.from_object(item)
20
- if item_expr is None or not (item_expr.col_type.is_string_type() or item_expr.col_type.is_image_type()):
21
- raise excs.Error(f'similarity(): requires a string or a PIL.Image.Image object, not a {type(item)}')
22
- assert item_expr.col_type.is_string_type() or item_expr.col_type.is_image_type()
22
+ """
23
+ A similarity expression against an embedding index.
24
+ """
23
25
 
24
- self.components = [col_ref, item_expr]
26
+ idx_id: int
27
+ idx_name: str
25
28
 
26
- from pixeltable import index
29
+ def __init__(self, col_ref: ColumnRef, item: Expr, idx_name: str | None = None):
30
+ from pixeltable.index import EmbeddingIndex
27
31
 
28
- # determine index to use
29
- idx_dict = col_ref.find_embedding_index(idx_name, 'similarity')
30
- assert len(idx_dict) == 1
31
- self.idx_info = next(iter(idx_dict.values()))
32
- idx = self.idx_info.idx
33
- assert isinstance(idx, index.EmbeddingIndex)
32
+ super().__init__(ts.FloatType())
34
33
 
35
- if item_expr.col_type.is_string_type() and idx.string_embed is None:
36
- raise excs.Error(
37
- f'Embedding index {self.idx_info.name!r} on column {self.idx_info.col.name!r} does not have a '
38
- f'string embedding and does not support string queries'
39
- )
40
- if item_expr.col_type.is_image_type() and idx.image_embed is None:
34
+ self.components = [col_ref, item]
35
+
36
+ # determine index to use
37
+ idx_info = col_ref.tbl.get().get_idx(col_ref.col, idx_name, EmbeddingIndex)
38
+ self.idx_id = idx_info.id
39
+ self.idx_name = idx_info.name
40
+ idx = idx_info.idx
41
+ assert isinstance(idx, EmbeddingIndex)
42
+
43
+ if item.col_type._type not in idx.embeddings:
44
+ type_str = item.col_type._type.name.lower()
45
+ article = 'an' if type_str[0] in 'aeiou' else 'a'
41
46
  raise excs.Error(
42
- f'Embedding index {self.idx_info.name!r} on column {self.idx_info.col.name!r} does not have an '
43
- f'image embedding and does not support image queries'
47
+ f'Embedding index {idx_info.name!r} on column {idx_info.col.name!r} does not have {article} '
48
+ f'{type_str} embedding and does not support {type_str} queries'
44
49
  )
45
50
  self.id = self._create_id()
46
51
 
@@ -48,38 +53,51 @@ class SimilarityExpr(Expr):
48
53
  return f'{self.components[0]}.similarity({self.components[1]})'
49
54
 
50
55
  def _id_attrs(self) -> list[tuple[str, Any]]:
51
- return [*super()._id_attrs(), ('idx_name', self.idx_info.name)]
56
+ return [*super()._id_attrs(), ('idx_id', self.idx_id)]
52
57
 
53
58
  def default_column_name(self) -> str:
54
59
  return 'similarity'
55
60
 
56
- def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
61
+ def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
62
+ from pixeltable.index import EmbeddingIndex
63
+
64
+ # check for a literal here, instead of the c'tor: needed for ExprTemplateFunctions
57
65
  if not isinstance(self.components[1], Literal):
58
- raise excs.Error('similarity(): requires a string or a PIL.Image.Image object, not an expression')
59
- item = self.components[1].val
60
- from pixeltable import index
66
+ raise excs.Error('similarity(): requires a value, not an expression')
67
+ idx_info = self._resolve_idx()
68
+ assert isinstance(idx_info.idx, EmbeddingIndex)
69
+ return idx_info.idx.similarity_clause(idx_info.val_col, self.components[1])
61
70
 
62
- assert isinstance(self.idx_info.idx, index.EmbeddingIndex)
63
- return self.idx_info.idx.similarity_clause(self.idx_info.val_col, item)
71
+ def as_order_by_clause(self, is_asc: bool) -> sql.ColumnElement | None:
72
+ from pixeltable.index import EmbeddingIndex
64
73
 
65
- def as_order_by_clause(self, is_asc: bool) -> Optional[sql.ColumnElement]:
74
+ # check for a literal here, instead of the c'tor: needed for ExprTemplateFunctions
66
75
  if not isinstance(self.components[1], Literal):
67
- raise excs.Error('similarity(): requires a string or a PIL.Image.Image object, not an expression')
68
- item = self.components[1].val
69
- from pixeltable import index
70
-
71
- assert isinstance(self.idx_info.idx, index.EmbeddingIndex)
72
- return self.idx_info.idx.order_by_clause(self.idx_info.val_col, item, is_asc)
76
+ raise excs.Error('similarity(): requires a value, not an expression')
77
+ idx_info = self._resolve_idx()
78
+ assert isinstance(idx_info.idx, EmbeddingIndex)
79
+ return idx_info.idx.order_by_clause(idx_info.val_col, self.components[1], is_asc)
80
+
81
+ def _resolve_idx(self) -> 'TableVersion.IndexInfo':
82
+ from pixeltable.index import EmbeddingIndex
83
+
84
+ # resolve idx_id
85
+ col_ref = self.components[0]
86
+ if self.idx_id not in col_ref.tbl.get().idxs:
87
+ raise excs.Error(f'Index {self.idx_name!r} not found')
88
+ idx_info = col_ref.tbl.get().idxs[self.idx_id]
89
+ assert isinstance(idx_info.idx, EmbeddingIndex)
90
+ return idx_info
73
91
 
74
92
  def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
75
93
  raise excs.Error('similarity(): cannot be used in a computed column')
76
94
 
77
95
  def _as_dict(self) -> dict:
78
- return {'idx_name': self.idx_info.name, **super()._as_dict()}
96
+ return {'idx_name': self.idx_name, **super()._as_dict()}
79
97
 
80
98
  @classmethod
81
99
  def _from_dict(cls, d: dict, components: list[Expr]) -> 'SimilarityExpr':
82
- iname = d.get('idx_name')
100
+ idx_name = d.get('idx_name')
83
101
  assert len(components) == 2
84
102
  assert isinstance(components[0], ColumnRef)
85
- return cls(components[0], components[1], idx_name=iname)
103
+ return cls(components[0], components[1], idx_name=idx_name)
@@ -1,4 +1,4 @@
1
- from typing import Iterable, Optional
1
+ from typing import Iterable
2
2
 
3
3
  import sqlalchemy as sql
4
4
 
@@ -9,9 +9,9 @@ from .expr_dict import ExprDict
9
9
  class SqlElementCache:
10
10
  """Cache of sql.ColumnElements for exprs"""
11
11
 
12
- cache: dict[int, Optional[sql.ColumnElement]] # key: Expr.id
12
+ cache: dict[int, sql.ColumnElement | None] # key: Expr.id
13
13
 
14
- def __init__(self, elements: Optional[ExprDict[sql.ColumnElement]] = None):
14
+ def __init__(self, elements: ExprDict[sql.ColumnElement] | None = None):
15
15
  self.cache = {}
16
16
  if elements is not None:
17
17
  for e, el in elements.items():
@@ -21,7 +21,7 @@ class SqlElementCache:
21
21
  for e, el in elements.items():
22
22
  self.cache[e.id] = el
23
23
 
24
- def get(self, e: Expr) -> Optional[sql.ColumnElement]:
24
+ def get(self, e: Expr) -> sql.ColumnElement | None:
25
25
  """Returns the sql.ColumnElement for the given Expr, or None if Expr.to_sql() returns None."""
26
26
  try:
27
27
  return self.cache[e.id]
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- from typing import Any, Optional, Union
3
+ from typing import Any
4
4
 
5
5
  import sqlalchemy as sql
6
6
 
@@ -60,7 +60,7 @@ class StringOp(Expr):
60
60
  def _id_attrs(self) -> list[tuple[str, Any]]:
61
61
  return [*super()._id_attrs(), ('operator', self.operator.value)]
62
62
 
63
- def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
63
+ def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
64
64
  left = sql_elements.get(self._op1)
65
65
  right = sql_elements.get(self._op2)
66
66
  if left is None or right is None:
@@ -68,7 +68,7 @@ class StringOp(Expr):
68
68
  if self.operator == StringOperator.CONCAT:
69
69
  return left.concat(right)
70
70
  if self.operator == StringOperator.REPEAT:
71
- return sql.func.repeat(sql.cast(left, sql.String), sql.cast(right, sql.Integer))
71
+ return sql.func.repeat(left.cast(sql.String), right.cast(sql.Integer))
72
72
  return None
73
73
 
74
74
  def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
@@ -76,7 +76,7 @@ class StringOp(Expr):
76
76
  op2_val = data_row[self._op2.slot_idx]
77
77
  data_row[self.slot_idx] = self.eval_nullable(op1_val, op2_val)
78
78
 
79
- def eval_nullable(self, op1_val: Union[str, None], op2_val: Union[int, str, None]) -> Union[str, None]:
79
+ def eval_nullable(self, op1_val: str | None, op2_val: int | str | None) -> str | None:
80
80
  """
81
81
  Return the result of evaluating the expression on two nullable int/float operands,
82
82
  None is interpreted as SQL NULL
@@ -85,7 +85,7 @@ class StringOp(Expr):
85
85
  return None
86
86
  return self.eval_non_null(op1_val, op2_val)
87
87
 
88
- def eval_non_null(self, op1_val: str, op2_val: Union[int, str]) -> str:
88
+ def eval_non_null(self, op1_val: str, op2_val: int | str) -> str:
89
89
  """
90
90
  Return the result of evaluating the expression on two int/float operands
91
91
  """
@@ -1,5 +1,3 @@
1
- from typing import Optional
2
-
3
1
  import sqlalchemy as sql
4
2
 
5
3
  from pixeltable import type_system as ts
@@ -19,7 +17,7 @@ class TypeCast(Expr):
19
17
  def __init__(self, underlying: Expr, new_type: ts.ColumnType):
20
18
  super().__init__(new_type)
21
19
  self.components: list[Expr] = [underlying]
22
- self.id: Optional[int] = self._create_id()
20
+ self.id: int | None = self._create_id()
23
21
 
24
22
  def _equals(self, other: 'TypeCast') -> bool:
25
23
  # `TypeCast` has no properties beyond those captured by `Expr`.
@@ -29,7 +27,7 @@ class TypeCast(Expr):
29
27
  def _op1(self) -> Expr:
30
28
  return self.components[0]
31
29
 
32
- def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
30
+ def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
33
31
  """
34
32
  sql_expr() is unimplemented for now, in order to sidestep potentially thorny
35
33
  questions about consistency of doing type conversions in both Python and Postgres.
@@ -40,7 +38,7 @@ class TypeCast(Expr):
40
38
  original_val = data_row[self._op1.slot_idx]
41
39
  data_row[self.slot_idx] = self.col_type.create_literal(original_val)
42
40
 
43
- def as_literal(self) -> Optional[Literal]:
41
+ def as_literal(self) -> Literal | None:
44
42
  op1_lit = self._op1.as_literal()
45
43
  if op1_lit is None:
46
44
  return None
@@ -5,6 +5,7 @@ from .callable_function import CallableFunction
5
5
  from .expr_template_function import ExprTemplateFunction
6
6
  from .function import Function, InvalidFunction
7
7
  from .function_registry import FunctionRegistry
8
+ from .mcp import mcp_udfs
8
9
  from .query_template_function import QueryTemplateFunction, query, retrieval_udf
9
10
  from .signature import Batch, Parameter, Signature
10
11
  from .tools import Tool, ToolChoice, Tools
@@ -2,7 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import abc
4
4
  import inspect
5
- from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Sequence, overload
5
+ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Sequence, overload
6
6
 
7
7
  import pixeltable.exceptions as excs
8
8
  import pixeltable.type_system as ts
@@ -42,7 +42,7 @@ class AggregateFunction(Function):
42
42
  def __init__(
43
43
  self,
44
44
  agg_class: type[Aggregator],
45
- type_substitutions: Optional[Sequence[dict]],
45
+ type_substitutions: Sequence[dict] | None,
46
46
  self_path: str,
47
47
  requires_order_by: bool,
48
48
  allows_std_agg: bool,
@@ -75,7 +75,7 @@ class AggregateFunction(Function):
75
75
  self.init_param_names = [self.init_param_names[signature_idx]]
76
76
 
77
77
  def __cls_to_signature(
78
- self, cls: type[Aggregator], type_substitutions: Optional[dict] = None
78
+ self, cls: type[Aggregator], type_substitutions: dict | None = None
79
79
  ) -> tuple[Signature, list[str]]:
80
80
  """Inspects the Aggregator class to infer the corresponding function signature. Returns the
81
81
  inferred signature along with the list of init_param_names (for downstream error handling).
@@ -159,7 +159,7 @@ class AggregateFunction(Function):
159
159
  self.init_param_names.append(init_param_names)
160
160
  return self
161
161
 
162
- def comment(self) -> Optional[str]:
162
+ def comment(self) -> str | None:
163
163
  return inspect.getdoc(self.agg_classes[0])
164
164
 
165
165
  def help_str(self) -> str:
@@ -173,7 +173,7 @@ class AggregateFunction(Function):
173
173
  from pixeltable import exprs
174
174
 
175
175
  # perform semantic analysis of special parameters 'order_by' and 'group_by'
176
- order_by_clause: Optional[Any] = None
176
+ order_by_clause: Any | None = None
177
177
  if self.ORDER_BY_PARAM in kwargs:
178
178
  if self.requires_order_by:
179
179
  raise excs.Error(
@@ -198,7 +198,7 @@ class AggregateFunction(Function):
198
198
  # don't pass the first parameter on, the Function doesn't get to see it
199
199
  args = args[1:]
200
200
 
201
- group_by_clause: Optional[Any] = None
201
+ group_by_clause: Any | None = None
202
202
  if self.GROUP_BY_PARAM in kwargs:
203
203
  if not self.allows_window:
204
204
  raise excs.Error(
@@ -248,7 +248,7 @@ def uda(
248
248
  requires_order_by: bool = False,
249
249
  allows_std_agg: bool = True,
250
250
  allows_window: bool = False,
251
- type_substitutions: Optional[Sequence[dict]] = None,
251
+ type_substitutions: Sequence[dict] | None = None,
252
252
  ) -> Callable[[type[Aggregator]], AggregateFunction]: ...
253
253
 
254
254
 
@@ -302,7 +302,7 @@ def make_aggregator(
302
302
  requires_order_by: bool = False,
303
303
  allows_std_agg: bool = True,
304
304
  allows_window: bool = False,
305
- type_substitutions: Optional[Sequence[dict]] = None,
305
+ type_substitutions: Sequence[dict] | None = None,
306
306
  ) -> AggregateFunction:
307
307
  class_path = f'{cls.__module__}.{cls.__qualname__}'
308
308
  instance = AggregateFunction(cls, type_substitutions, class_path, requires_order_by, allows_std_agg, allows_window)
@@ -1,7 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import inspect
4
- from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence
4
+ from typing import TYPE_CHECKING, Any, Callable, Sequence
5
5
  from uuid import UUID
6
6
 
7
7
  import cloudpickle # type: ignore[import-untyped]
@@ -25,16 +25,16 @@ class CallableFunction(Function):
25
25
  """
26
26
 
27
27
  py_fns: list[Callable]
28
- self_name: Optional[str]
29
- batch_size: Optional[int]
28
+ self_name: str | None
29
+ batch_size: int | None
30
30
 
31
31
  def __init__(
32
32
  self,
33
33
  signatures: list[Signature],
34
34
  py_fns: list[Callable],
35
- self_path: Optional[str] = None,
36
- self_name: Optional[str] = None,
37
- batch_size: Optional[int] = None,
35
+ self_path: str | None = None,
36
+ self_name: str | None = None,
37
+ batch_size: int | None = None,
38
38
  is_method: bool = False,
39
39
  is_property: bool = False,
40
40
  ):
@@ -60,7 +60,7 @@ class CallableFunction(Function):
60
60
  def is_async(self) -> bool:
61
61
  return inspect.iscoroutinefunction(self.py_fn)
62
62
 
63
- def comment(self) -> Optional[str]:
63
+ def comment(self) -> str | None:
64
64
  return inspect.getdoc(self.py_fns[0])
65
65
 
66
66
  @property
@@ -138,7 +138,7 @@ class CallableFunction(Function):
138
138
  batched_kwargs = {k: v for k, v in kwargs.items() if k not in constant_param_names}
139
139
  return constant_kwargs, batched_kwargs
140
140
 
141
- def get_batch_size(self, *args: Any, **kwargs: Any) -> Optional[int]:
141
+ def get_batch_size(self, *args: Any, **kwargs: Any) -> int | None:
142
142
  return self.batch_size
143
143
 
144
144
  @property
@@ -187,7 +187,7 @@ class CallableFunction(Function):
187
187
  return md, cloudpickle.dumps(self.py_fn)
188
188
 
189
189
  @classmethod
190
- def from_store(cls, name: Optional[str], md: dict, binary_obj: bytes) -> Function:
190
+ def from_store(cls, name: str | None, md: dict, binary_obj: bytes) -> Function:
191
191
  py_fn = cloudpickle.loads(binary_obj)
192
192
  assert callable(py_fn)
193
193
  sig = Signature.from_dict(md['signature'])
@@ -1,4 +1,4 @@
1
- from typing import Any, Optional, Sequence
1
+ from typing import Any, Sequence
2
2
 
3
3
  from pixeltable import exceptions as excs, exprs, type_system as ts
4
4
 
@@ -34,6 +34,9 @@ class ExprTemplate:
34
34
  continue
35
35
  self.defaults[param.name] = param.default
36
36
 
37
+ # def __repr__(self) -> str:
38
+ # return f'ExprTemplate(expr={repr(self.expr)}, signature={self.signature})'
39
+
37
40
 
38
41
  class ExprTemplateFunction(Function):
39
42
  """A parameterized expression from which an executable Expr is created with a function call."""
@@ -41,7 +44,7 @@ class ExprTemplateFunction(Function):
41
44
  templates: list[ExprTemplate]
42
45
  self_name: str
43
46
 
44
- def __init__(self, templates: list[ExprTemplate], self_path: Optional[str] = None, name: Optional[str] = None):
47
+ def __init__(self, templates: list[ExprTemplate], self_path: str | None = None, name: str | None = None):
45
48
  self.templates = templates
46
49
  self.self_name = name
47
50
 
@@ -59,7 +62,7 @@ class ExprTemplateFunction(Function):
59
62
  assert not self.is_polymorphic
60
63
  template = self.template
61
64
  bound_args = self.signature.py_signature.bind(*args, **kwargs).arguments
62
- # apply defaults, otherwise we might have Parameters left over
65
+ # apply defaults; otherwise we might have Parameters left over
63
66
  bound_args.update(
64
67
  {param_name: default for param_name, default in template.defaults.items() if param_name not in bound_args}
65
68
  )
@@ -85,29 +88,29 @@ class ExprTemplateFunction(Function):
85
88
  conditional_return_type).
86
89
  """
87
90
  assert not self.is_polymorphic
88
- template = self.template
89
91
  with_defaults = bound_args.copy()
90
92
  with_defaults.update(
91
- {param_name: default for param_name, default in template.defaults.items() if param_name not in bound_args}
93
+ {
94
+ param_name: default
95
+ for param_name, default in self.template.defaults.items()
96
+ if param_name not in bound_args
97
+ }
92
98
  )
93
99
  substituted_expr = self.template.expr.copy().substitute(
94
- {template.param_exprs[name]: expr for name, expr in with_defaults.items()}
100
+ {self.template.param_exprs[name]: expr for name, expr in with_defaults.items()}
95
101
  )
96
102
  return substituted_expr.col_type
97
103
 
98
- def comment(self) -> Optional[str]:
104
+ def comment(self) -> str | None:
99
105
  if isinstance(self.templates[0].expr, exprs.FunctionCall):
100
106
  return self.templates[0].expr.fn.comment()
101
107
  return None
102
108
 
103
109
  def exec(self, args: Sequence[Any], kwargs: dict[str, Any]) -> Any:
104
- from pixeltable import exec
105
-
106
110
  assert not self.is_polymorphic
107
111
  expr = self.instantiate(args, kwargs)
108
112
  row_builder = exprs.RowBuilder(output_exprs=[expr], columns=[], input_exprs=[])
109
- row_batch = exec.DataRowBatch(tbl=None, row_builder=row_builder, num_rows=1)
110
- row = row_batch[0]
113
+ row = row_builder.make_row()
111
114
  row_builder.eval(row, ctx=row_builder.default_eval_ctx)
112
115
  return row[row_builder.get_output_exprs()[0].slot_idx]
113
116
 
@@ -130,6 +133,9 @@ class ExprTemplateFunction(Function):
130
133
  def __str__(self) -> str:
131
134
  return str(self.templates[0].expr)
132
135
 
136
+ # def __repr__(self) -> str:
137
+ # return f'ExprTemplateFunction(name={self.name}, templates={self.templates})'
138
+
133
139
  def _as_dict(self) -> dict:
134
140
  if self.self_path is not None:
135
141
  return super()._as_dict()
@@ -5,7 +5,7 @@ import inspect
5
5
  import typing
6
6
  from abc import ABC, abstractmethod
7
7
  from copy import copy
8
- from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence, cast
8
+ from typing import TYPE_CHECKING, Any, Callable, Sequence, cast
9
9
 
10
10
  import sqlalchemy as sql
11
11
  from typing_extensions import Self
@@ -30,10 +30,10 @@ class Function(ABC):
30
30
  """
31
31
 
32
32
  signatures: list[Signature]
33
- self_path: Optional[str]
33
+ self_path: str | None
34
34
  is_method: bool
35
35
  is_property: bool
36
- _conditional_return_type: Optional[Callable[..., ts.ColumnType]]
36
+ _conditional_return_type: Callable[..., ts.ColumnType] | None
37
37
 
38
38
  # We cache the overload resolutions in self._resolutions. This ensures that each resolution is represented
39
39
  # globally by a single Python object. We do this dynamically rather than pre-constructing them in order to
@@ -43,17 +43,17 @@ class Function(ABC):
43
43
  # Translates a call to this function with the given arguments to its SQLAlchemy equivalent.
44
44
  # Overriden for specific Function instances via the to_sql() decorator. The override must accept the same
45
45
  # parameter names as the original function. Each parameter is going to be of type sql.ColumnElement.
46
- _to_sql: Callable[..., Optional[sql.ColumnElement]]
46
+ _to_sql: Callable[..., sql.ColumnElement | None]
47
47
 
48
48
  # Returns the resource pool to use for calling this function with the given arguments.
49
49
  # Overriden for specific Function instances via the resource_pool() decorator. The override must accept a subset
50
50
  # of the parameters of the original function, with the same type.
51
- _resource_pool: Callable[..., Optional[str]]
51
+ _resource_pool: Callable[..., str | None]
52
52
 
53
53
  def __init__(
54
54
  self,
55
55
  signatures: list[Signature],
56
- self_path: Optional[str] = None,
56
+ self_path: str | None = None,
57
57
  is_method: bool = False,
58
58
  is_property: bool = False,
59
59
  ):
@@ -105,7 +105,7 @@ class Function(ABC):
105
105
  @abstractmethod
106
106
  def is_async(self) -> bool: ...
107
107
 
108
- def comment(self) -> Optional[str]:
108
+ def comment(self) -> str | None:
109
109
  return None
110
110
 
111
111
  def help_str(self) -> str:
@@ -175,7 +175,7 @@ class Function(ABC):
175
175
 
176
176
  def _bind_to_matching_signature(self, args: Sequence[Any], kwargs: dict[str, Any]) -> tuple[Self, dict[str, Any]]:
177
177
  result: int = -1
178
- bound_args: Optional[dict[str, Any]] = None
178
+ bound_args: dict[str, Any] | None = None
179
179
  assert len(self.signatures) > 0
180
180
  if len(self.signatures) == 1:
181
181
  # Only one signature: call _bind_to_signature() and surface any errors directly
@@ -206,7 +206,7 @@ class Function(ABC):
206
206
  self._resolved_fns[signature_idx].validate_call(normalized_args)
207
207
  return normalized_args
208
208
 
209
- def validate_call(self, bound_args: dict[str, Optional['exprs.Expr']]) -> None:
209
+ def validate_call(self, bound_args: dict[str, 'exprs.Expr' | None]) -> None:
210
210
  """Override this to do custom validation of the arguments"""
211
211
  assert not self.is_polymorphic
212
212
  self.signature.validate_args(bound_args, context=f'in function {self.name!r}')
@@ -252,9 +252,7 @@ class Function(ABC):
252
252
 
253
253
  return return_type
254
254
 
255
- def _assemble_callable_args(
256
- self, callable: Callable, bound_args: dict[str, 'exprs.Expr']
257
- ) -> Optional[dict[str, Any]]:
255
+ def _assemble_callable_args(self, callable: Callable, bound_args: dict[str, 'exprs.Expr']) -> dict[str, Any] | None:
258
256
  """
259
257
  Return the kwargs to pass to callable, given bound_args passed to this function.
260
258
 
@@ -286,7 +284,7 @@ class Function(ABC):
286
284
  return None
287
285
  assert isinstance(arg, exprs.Expr)
288
286
 
289
- expects_expr: Optional[type[exprs.Expr]] = None
287
+ expects_expr: type[exprs.Expr] | None = None
290
288
  type_hint = callable_type_hints.get(param.name)
291
289
  if typing.get_origin(type_hint) is not None:
292
290
  type_hint = typing.get_origin(type_hint) # Remove type subscript if one exists
@@ -410,12 +408,12 @@ class Function(ABC):
410
408
  """Execute the function with the given arguments and return the result."""
411
409
  raise NotImplementedError()
412
410
 
413
- def to_sql(self, fn: Callable[..., Optional[sql.ColumnElement]]) -> Callable[..., Optional[sql.ColumnElement]]:
411
+ def to_sql(self, fn: Callable[..., sql.ColumnElement | None]) -> Callable[..., sql.ColumnElement | None]:
414
412
  """Instance decorator for specifying the SQL translation of this function"""
415
413
  self._to_sql = fn
416
414
  return fn
417
415
 
418
- def __default_to_sql(self, *args: Any, **kwargs: Any) -> Optional[sql.ColumnElement]:
416
+ def __default_to_sql(self, *args: Any, **kwargs: Any) -> sql.ColumnElement | None:
419
417
  """The default implementation of SQL translation, which provides no translation"""
420
418
  return None
421
419
 
@@ -425,7 +423,7 @@ class Function(ABC):
425
423
  self._resource_pool = fn
426
424
  return fn
427
425
 
428
- def __default_resource_pool(self) -> Optional[str]:
426
+ def __default_resource_pool(self) -> str | None:
429
427
  return None
430
428
 
431
429
  def __eq__(self, other: object) -> bool:
@@ -495,7 +493,7 @@ class Function(ABC):
495
493
  raise NotImplementedError()
496
494
 
497
495
  @classmethod
498
- def from_store(cls, name: Optional[str], md: dict, binary_obj: bytes) -> Function:
496
+ def from_store(cls, name: str | None, md: dict, binary_obj: bytes) -> Function:
499
497
  """
500
498
  Create a Function instance from the serialized representation returned by to_store()
501
499
  """
@@ -504,12 +502,12 @@ class Function(ABC):
504
502
 
505
503
  class InvalidFunction(Function):
506
504
  fn_dict: dict[str, Any]
507
- errormsg: str
505
+ error_msg: str
508
506
 
509
- def __init__(self, self_path: str, fn_dict: dict[str, Any], errormsg: str):
507
+ def __init__(self, self_path: str, fn_dict: dict[str, Any], error_msg: str):
510
508
  super().__init__([], self_path)
511
509
  self.fn_dict = fn_dict
512
- self.errormsg = errormsg
510
+ self.error_msg = error_msg
513
511
 
514
512
  def _as_dict(self) -> dict:
515
513
  """
@@ -4,7 +4,6 @@ import dataclasses
4
4
  import importlib
5
5
  import logging
6
6
  import sys
7
- from typing import Optional
8
7
  from uuid import UUID
9
8
 
10
9
  import sqlalchemy as sql
@@ -23,7 +22,7 @@ class FunctionRegistry:
23
22
  Function are loaded from the store on demand.
24
23
  """
25
24
 
26
- _instance: Optional[FunctionRegistry] = None
25
+ _instance: FunctionRegistry | None = None
27
26
 
28
27
  @classmethod
29
28
  def get(cls) -> FunctionRegistry:
@@ -94,7 +93,7 @@ class FunctionRegistry:
94
93
  # stored_fn_md.append(md)
95
94
  return list(self.module_fns.values())
96
95
 
97
- # def get_function(self, *, id: Optional[UUID] = None, fqn: Optional[str] = None) -> Function:
96
+ # def get_function(self, *, id: UUID | None = None, fqn: str | None = None) -> Function:
98
97
  # assert (id is not None) != (fqn is not None)
99
98
  # if id is not None:
100
99
  # if id not in self.stored_fns_by_id:
@@ -143,7 +142,7 @@ class FunctionRegistry:
143
142
  return list(self.type_methods[base_type].values())
144
143
  return []
145
144
 
146
- def lookup_type_method(self, base_type: ts.ColumnType.Type, name: str) -> Optional[Function]:
145
+ def lookup_type_method(self, base_type: ts.ColumnType.Type, name: str) -> Function | None:
147
146
  """
148
147
  Look up a method (or property) by name for a given base type. If no such method is registered, return None.
149
148
  """
@@ -151,8 +150,8 @@ class FunctionRegistry:
151
150
  return self.type_methods[base_type][name]
152
151
  return None
153
152
 
154
- # def create_function(self, md: schema.FunctionMd, binary_obj: bytes, dir_id: Optional[UUID] = None) -> UUID:
155
- def create_stored_function(self, pxt_fn: Function, dir_id: Optional[UUID] = None) -> UUID:
153
+ # def create_function(self, md: schema.FunctionMd, binary_obj: bytes, dir_id: UUID | None = None) -> UUID:
154
+ def create_stored_function(self, pxt_fn: Function, dir_id: UUID | None = None) -> UUID:
156
155
  fn_md, binary_obj = pxt_fn.to_store()
157
156
  md = schema.FunctionMd(name=pxt_fn.name, md=fn_md, py_version=sys.version, class_name=pxt_fn.__class__.__name__)
158
157
  with env.Env.get().engine.begin() as conn:
@@ -184,7 +183,7 @@ class FunctionRegistry:
184
183
  self.stored_fns_by_id[id] = instance
185
184
  return instance
186
185
 
187
- # def create_function(self, fn: Function, dir_id: Optional[UUID] = None, name: Optional[str] = None) -> None:
186
+ # def create_function(self, fn: Function, dir_id: UUID | None = None, name: str | None = None) -> None:
188
187
  # with env.Env.get().engine.begin() as conn:
189
188
  # _logger.debug(f'Pickling function {name}')
190
189
  # eval_fn_str = cloudpickle.dumps(fn.eval_fn) if fn.eval_fn is not None else None
@@ -1,14 +1,13 @@
1
1
  import importlib
2
2
  import inspect
3
3
  from types import ModuleType
4
- from typing import Optional
5
4
 
6
5
  import pixeltable.exceptions as excs
7
6
 
8
7
 
9
- def resolve_symbol(symbol_path: str) -> Optional[object]:
8
+ def resolve_symbol(symbol_path: str) -> object | None:
10
9
  path_elems = symbol_path.split('.')
11
- module: Optional[ModuleType] = None
10
+ module: ModuleType | None = None
12
11
  i = len(path_elems) - 1
13
12
  while i > 0 and module is None:
14
13
  try: