pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pixeltable might be problematic. Click here for more details.

Files changed (202) hide show
  1. pixeltable/__init__.py +23 -5
  2. pixeltable/_version.py +1 -0
  3. pixeltable/catalog/__init__.py +5 -3
  4. pixeltable/catalog/catalog.py +1318 -404
  5. pixeltable/catalog/column.py +186 -115
  6. pixeltable/catalog/dir.py +1 -2
  7. pixeltable/catalog/globals.py +11 -43
  8. pixeltable/catalog/insertable_table.py +167 -79
  9. pixeltable/catalog/path.py +61 -23
  10. pixeltable/catalog/schema_object.py +9 -10
  11. pixeltable/catalog/table.py +626 -308
  12. pixeltable/catalog/table_metadata.py +101 -0
  13. pixeltable/catalog/table_version.py +713 -569
  14. pixeltable/catalog/table_version_handle.py +37 -6
  15. pixeltable/catalog/table_version_path.py +42 -29
  16. pixeltable/catalog/tbl_ops.py +50 -0
  17. pixeltable/catalog/update_status.py +191 -0
  18. pixeltable/catalog/view.py +108 -94
  19. pixeltable/config.py +128 -22
  20. pixeltable/dataframe.py +188 -100
  21. pixeltable/env.py +407 -136
  22. pixeltable/exceptions.py +6 -0
  23. pixeltable/exec/__init__.py +3 -0
  24. pixeltable/exec/aggregation_node.py +7 -8
  25. pixeltable/exec/cache_prefetch_node.py +83 -110
  26. pixeltable/exec/cell_materialization_node.py +231 -0
  27. pixeltable/exec/cell_reconstruction_node.py +135 -0
  28. pixeltable/exec/component_iteration_node.py +4 -3
  29. pixeltable/exec/data_row_batch.py +8 -65
  30. pixeltable/exec/exec_context.py +16 -4
  31. pixeltable/exec/exec_node.py +13 -36
  32. pixeltable/exec/expr_eval/evaluators.py +7 -6
  33. pixeltable/exec/expr_eval/expr_eval_node.py +27 -12
  34. pixeltable/exec/expr_eval/globals.py +8 -5
  35. pixeltable/exec/expr_eval/row_buffer.py +1 -2
  36. pixeltable/exec/expr_eval/schedulers.py +190 -30
  37. pixeltable/exec/globals.py +32 -0
  38. pixeltable/exec/in_memory_data_node.py +18 -18
  39. pixeltable/exec/object_store_save_node.py +293 -0
  40. pixeltable/exec/row_update_node.py +16 -9
  41. pixeltable/exec/sql_node.py +206 -101
  42. pixeltable/exprs/__init__.py +1 -1
  43. pixeltable/exprs/arithmetic_expr.py +27 -22
  44. pixeltable/exprs/array_slice.py +3 -3
  45. pixeltable/exprs/column_property_ref.py +34 -30
  46. pixeltable/exprs/column_ref.py +92 -96
  47. pixeltable/exprs/comparison.py +5 -5
  48. pixeltable/exprs/compound_predicate.py +5 -4
  49. pixeltable/exprs/data_row.py +152 -55
  50. pixeltable/exprs/expr.py +62 -43
  51. pixeltable/exprs/expr_dict.py +3 -3
  52. pixeltable/exprs/expr_set.py +17 -10
  53. pixeltable/exprs/function_call.py +75 -37
  54. pixeltable/exprs/globals.py +1 -2
  55. pixeltable/exprs/in_predicate.py +4 -4
  56. pixeltable/exprs/inline_expr.py +10 -27
  57. pixeltable/exprs/is_null.py +1 -3
  58. pixeltable/exprs/json_mapper.py +8 -8
  59. pixeltable/exprs/json_path.py +56 -22
  60. pixeltable/exprs/literal.py +5 -5
  61. pixeltable/exprs/method_ref.py +2 -2
  62. pixeltable/exprs/object_ref.py +2 -2
  63. pixeltable/exprs/row_builder.py +127 -53
  64. pixeltable/exprs/rowid_ref.py +8 -12
  65. pixeltable/exprs/similarity_expr.py +50 -25
  66. pixeltable/exprs/sql_element_cache.py +4 -4
  67. pixeltable/exprs/string_op.py +5 -5
  68. pixeltable/exprs/type_cast.py +3 -5
  69. pixeltable/func/__init__.py +1 -0
  70. pixeltable/func/aggregate_function.py +8 -8
  71. pixeltable/func/callable_function.py +9 -9
  72. pixeltable/func/expr_template_function.py +10 -10
  73. pixeltable/func/function.py +18 -20
  74. pixeltable/func/function_registry.py +6 -7
  75. pixeltable/func/globals.py +2 -3
  76. pixeltable/func/mcp.py +74 -0
  77. pixeltable/func/query_template_function.py +20 -18
  78. pixeltable/func/signature.py +43 -16
  79. pixeltable/func/tools.py +23 -13
  80. pixeltable/func/udf.py +18 -20
  81. pixeltable/functions/__init__.py +6 -0
  82. pixeltable/functions/anthropic.py +93 -33
  83. pixeltable/functions/audio.py +114 -10
  84. pixeltable/functions/bedrock.py +13 -6
  85. pixeltable/functions/date.py +1 -1
  86. pixeltable/functions/deepseek.py +20 -9
  87. pixeltable/functions/fireworks.py +2 -2
  88. pixeltable/functions/gemini.py +28 -11
  89. pixeltable/functions/globals.py +13 -13
  90. pixeltable/functions/groq.py +108 -0
  91. pixeltable/functions/huggingface.py +1046 -23
  92. pixeltable/functions/image.py +9 -18
  93. pixeltable/functions/llama_cpp.py +23 -8
  94. pixeltable/functions/math.py +3 -4
  95. pixeltable/functions/mistralai.py +4 -15
  96. pixeltable/functions/ollama.py +16 -9
  97. pixeltable/functions/openai.py +104 -82
  98. pixeltable/functions/openrouter.py +143 -0
  99. pixeltable/functions/replicate.py +2 -2
  100. pixeltable/functions/reve.py +250 -0
  101. pixeltable/functions/string.py +21 -28
  102. pixeltable/functions/timestamp.py +13 -14
  103. pixeltable/functions/together.py +4 -6
  104. pixeltable/functions/twelvelabs.py +92 -0
  105. pixeltable/functions/util.py +6 -1
  106. pixeltable/functions/video.py +1388 -106
  107. pixeltable/functions/vision.py +7 -7
  108. pixeltable/functions/whisper.py +15 -7
  109. pixeltable/functions/whisperx.py +179 -0
  110. pixeltable/{ext/functions → functions}/yolox.py +2 -4
  111. pixeltable/globals.py +332 -105
  112. pixeltable/index/base.py +13 -22
  113. pixeltable/index/btree.py +23 -22
  114. pixeltable/index/embedding_index.py +32 -44
  115. pixeltable/io/__init__.py +4 -2
  116. pixeltable/io/datarows.py +7 -6
  117. pixeltable/io/external_store.py +49 -77
  118. pixeltable/io/fiftyone.py +11 -11
  119. pixeltable/io/globals.py +29 -28
  120. pixeltable/io/hf_datasets.py +17 -9
  121. pixeltable/io/label_studio.py +70 -66
  122. pixeltable/io/lancedb.py +3 -0
  123. pixeltable/io/pandas.py +12 -11
  124. pixeltable/io/parquet.py +13 -93
  125. pixeltable/io/table_data_conduit.py +71 -47
  126. pixeltable/io/utils.py +3 -3
  127. pixeltable/iterators/__init__.py +2 -1
  128. pixeltable/iterators/audio.py +21 -11
  129. pixeltable/iterators/document.py +116 -55
  130. pixeltable/iterators/image.py +5 -2
  131. pixeltable/iterators/video.py +293 -13
  132. pixeltable/metadata/__init__.py +4 -2
  133. pixeltable/metadata/converters/convert_18.py +2 -2
  134. pixeltable/metadata/converters/convert_19.py +2 -2
  135. pixeltable/metadata/converters/convert_20.py +2 -2
  136. pixeltable/metadata/converters/convert_21.py +2 -2
  137. pixeltable/metadata/converters/convert_22.py +2 -2
  138. pixeltable/metadata/converters/convert_24.py +2 -2
  139. pixeltable/metadata/converters/convert_25.py +2 -2
  140. pixeltable/metadata/converters/convert_26.py +2 -2
  141. pixeltable/metadata/converters/convert_29.py +4 -4
  142. pixeltable/metadata/converters/convert_34.py +2 -2
  143. pixeltable/metadata/converters/convert_36.py +2 -2
  144. pixeltable/metadata/converters/convert_37.py +15 -0
  145. pixeltable/metadata/converters/convert_38.py +39 -0
  146. pixeltable/metadata/converters/convert_39.py +124 -0
  147. pixeltable/metadata/converters/convert_40.py +73 -0
  148. pixeltable/metadata/converters/util.py +13 -12
  149. pixeltable/metadata/notes.py +4 -0
  150. pixeltable/metadata/schema.py +79 -42
  151. pixeltable/metadata/utils.py +74 -0
  152. pixeltable/mypy/__init__.py +3 -0
  153. pixeltable/mypy/mypy_plugin.py +123 -0
  154. pixeltable/plan.py +274 -223
  155. pixeltable/share/__init__.py +1 -1
  156. pixeltable/share/packager.py +259 -129
  157. pixeltable/share/protocol/__init__.py +34 -0
  158. pixeltable/share/protocol/common.py +170 -0
  159. pixeltable/share/protocol/operation_types.py +33 -0
  160. pixeltable/share/protocol/replica.py +109 -0
  161. pixeltable/share/publish.py +213 -57
  162. pixeltable/store.py +238 -175
  163. pixeltable/type_system.py +104 -63
  164. pixeltable/utils/__init__.py +2 -3
  165. pixeltable/utils/arrow.py +108 -13
  166. pixeltable/utils/av.py +298 -0
  167. pixeltable/utils/azure_store.py +305 -0
  168. pixeltable/utils/code.py +3 -3
  169. pixeltable/utils/console_output.py +4 -1
  170. pixeltable/utils/coroutine.py +6 -23
  171. pixeltable/utils/dbms.py +31 -5
  172. pixeltable/utils/description_helper.py +4 -5
  173. pixeltable/utils/documents.py +5 -6
  174. pixeltable/utils/exception_handler.py +7 -30
  175. pixeltable/utils/filecache.py +6 -6
  176. pixeltable/utils/formatter.py +4 -6
  177. pixeltable/utils/gcs_store.py +283 -0
  178. pixeltable/utils/http_server.py +2 -3
  179. pixeltable/utils/iceberg.py +1 -2
  180. pixeltable/utils/image.py +17 -0
  181. pixeltable/utils/lancedb.py +88 -0
  182. pixeltable/utils/local_store.py +316 -0
  183. pixeltable/utils/misc.py +5 -0
  184. pixeltable/utils/object_stores.py +528 -0
  185. pixeltable/utils/pydantic.py +60 -0
  186. pixeltable/utils/pytorch.py +5 -6
  187. pixeltable/utils/s3_store.py +392 -0
  188. pixeltable-0.4.20.dist-info/METADATA +587 -0
  189. pixeltable-0.4.20.dist-info/RECORD +218 -0
  190. {pixeltable-0.4.0rc3.dist-info → pixeltable-0.4.20.dist-info}/WHEEL +1 -1
  191. pixeltable-0.4.20.dist-info/entry_points.txt +2 -0
  192. pixeltable/__version__.py +0 -3
  193. pixeltable/ext/__init__.py +0 -17
  194. pixeltable/ext/functions/__init__.py +0 -11
  195. pixeltable/ext/functions/whisperx.py +0 -77
  196. pixeltable/utils/media_store.py +0 -77
  197. pixeltable/utils/s3.py +0 -17
  198. pixeltable/utils/sample.py +0 -25
  199. pixeltable-0.4.0rc3.dist-info/METADATA +0 -435
  200. pixeltable-0.4.0rc3.dist-info/RECORD +0 -189
  201. pixeltable-0.4.0rc3.dist-info/entry_points.txt +0 -3
  202. {pixeltable-0.4.0rc3.dist-info → pixeltable-0.4.20.dist-info/licenses}/LICENSE +0 -0
@@ -1,5 +1,3 @@
1
- from typing import Optional
2
-
3
1
  import sqlalchemy as sql
4
2
 
5
3
  from pixeltable import type_system as ts
@@ -19,7 +17,7 @@ class TypeCast(Expr):
19
17
  def __init__(self, underlying: Expr, new_type: ts.ColumnType):
20
18
  super().__init__(new_type)
21
19
  self.components: list[Expr] = [underlying]
22
- self.id: Optional[int] = self._create_id()
20
+ self.id: int | None = self._create_id()
23
21
 
24
22
  def _equals(self, other: 'TypeCast') -> bool:
25
23
  # `TypeCast` has no properties beyond those captured by `Expr`.
@@ -29,7 +27,7 @@ class TypeCast(Expr):
29
27
  def _op1(self) -> Expr:
30
28
  return self.components[0]
31
29
 
32
- def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
30
+ def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
33
31
  """
34
32
  sql_expr() is unimplemented for now, in order to sidestep potentially thorny
35
33
  questions about consistency of doing type conversions in both Python and Postgres.
@@ -40,7 +38,7 @@ class TypeCast(Expr):
40
38
  original_val = data_row[self._op1.slot_idx]
41
39
  data_row[self.slot_idx] = self.col_type.create_literal(original_val)
42
40
 
43
- def as_literal(self) -> Optional[Literal]:
41
+ def as_literal(self) -> Literal | None:
44
42
  op1_lit = self._op1.as_literal()
45
43
  if op1_lit is None:
46
44
  return None
@@ -5,6 +5,7 @@ from .callable_function import CallableFunction
5
5
  from .expr_template_function import ExprTemplateFunction
6
6
  from .function import Function, InvalidFunction
7
7
  from .function_registry import FunctionRegistry
8
+ from .mcp import mcp_udfs
8
9
  from .query_template_function import QueryTemplateFunction, query, retrieval_udf
9
10
  from .signature import Batch, Parameter, Signature
10
11
  from .tools import Tool, ToolChoice, Tools
@@ -2,7 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import abc
4
4
  import inspect
5
- from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Sequence, overload
5
+ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Sequence, overload
6
6
 
7
7
  import pixeltable.exceptions as excs
8
8
  import pixeltable.type_system as ts
@@ -42,7 +42,7 @@ class AggregateFunction(Function):
42
42
  def __init__(
43
43
  self,
44
44
  agg_class: type[Aggregator],
45
- type_substitutions: Optional[Sequence[dict]],
45
+ type_substitutions: Sequence[dict] | None,
46
46
  self_path: str,
47
47
  requires_order_by: bool,
48
48
  allows_std_agg: bool,
@@ -75,7 +75,7 @@ class AggregateFunction(Function):
75
75
  self.init_param_names = [self.init_param_names[signature_idx]]
76
76
 
77
77
  def __cls_to_signature(
78
- self, cls: type[Aggregator], type_substitutions: Optional[dict] = None
78
+ self, cls: type[Aggregator], type_substitutions: dict | None = None
79
79
  ) -> tuple[Signature, list[str]]:
80
80
  """Inspects the Aggregator class to infer the corresponding function signature. Returns the
81
81
  inferred signature along with the list of init_param_names (for downstream error handling).
@@ -159,7 +159,7 @@ class AggregateFunction(Function):
159
159
  self.init_param_names.append(init_param_names)
160
160
  return self
161
161
 
162
- def comment(self) -> Optional[str]:
162
+ def comment(self) -> str | None:
163
163
  return inspect.getdoc(self.agg_classes[0])
164
164
 
165
165
  def help_str(self) -> str:
@@ -173,7 +173,7 @@ class AggregateFunction(Function):
173
173
  from pixeltable import exprs
174
174
 
175
175
  # perform semantic analysis of special parameters 'order_by' and 'group_by'
176
- order_by_clause: Optional[Any] = None
176
+ order_by_clause: Any | None = None
177
177
  if self.ORDER_BY_PARAM in kwargs:
178
178
  if self.requires_order_by:
179
179
  raise excs.Error(
@@ -198,7 +198,7 @@ class AggregateFunction(Function):
198
198
  # don't pass the first parameter on, the Function doesn't get to see it
199
199
  args = args[1:]
200
200
 
201
- group_by_clause: Optional[Any] = None
201
+ group_by_clause: Any | None = None
202
202
  if self.GROUP_BY_PARAM in kwargs:
203
203
  if not self.allows_window:
204
204
  raise excs.Error(
@@ -248,7 +248,7 @@ def uda(
248
248
  requires_order_by: bool = False,
249
249
  allows_std_agg: bool = True,
250
250
  allows_window: bool = False,
251
- type_substitutions: Optional[Sequence[dict]] = None,
251
+ type_substitutions: Sequence[dict] | None = None,
252
252
  ) -> Callable[[type[Aggregator]], AggregateFunction]: ...
253
253
 
254
254
 
@@ -302,7 +302,7 @@ def make_aggregator(
302
302
  requires_order_by: bool = False,
303
303
  allows_std_agg: bool = True,
304
304
  allows_window: bool = False,
305
- type_substitutions: Optional[Sequence[dict]] = None,
305
+ type_substitutions: Sequence[dict] | None = None,
306
306
  ) -> AggregateFunction:
307
307
  class_path = f'{cls.__module__}.{cls.__qualname__}'
308
308
  instance = AggregateFunction(cls, type_substitutions, class_path, requires_order_by, allows_std_agg, allows_window)
@@ -1,7 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import inspect
4
- from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence
4
+ from typing import TYPE_CHECKING, Any, Callable, Sequence
5
5
  from uuid import UUID
6
6
 
7
7
  import cloudpickle # type: ignore[import-untyped]
@@ -25,16 +25,16 @@ class CallableFunction(Function):
25
25
  """
26
26
 
27
27
  py_fns: list[Callable]
28
- self_name: Optional[str]
29
- batch_size: Optional[int]
28
+ self_name: str | None
29
+ batch_size: int | None
30
30
 
31
31
  def __init__(
32
32
  self,
33
33
  signatures: list[Signature],
34
34
  py_fns: list[Callable],
35
- self_path: Optional[str] = None,
36
- self_name: Optional[str] = None,
37
- batch_size: Optional[int] = None,
35
+ self_path: str | None = None,
36
+ self_name: str | None = None,
37
+ batch_size: int | None = None,
38
38
  is_method: bool = False,
39
39
  is_property: bool = False,
40
40
  ):
@@ -60,7 +60,7 @@ class CallableFunction(Function):
60
60
  def is_async(self) -> bool:
61
61
  return inspect.iscoroutinefunction(self.py_fn)
62
62
 
63
- def comment(self) -> Optional[str]:
63
+ def comment(self) -> str | None:
64
64
  return inspect.getdoc(self.py_fns[0])
65
65
 
66
66
  @property
@@ -138,7 +138,7 @@ class CallableFunction(Function):
138
138
  batched_kwargs = {k: v for k, v in kwargs.items() if k not in constant_param_names}
139
139
  return constant_kwargs, batched_kwargs
140
140
 
141
- def get_batch_size(self, *args: Any, **kwargs: Any) -> Optional[int]:
141
+ def get_batch_size(self, *args: Any, **kwargs: Any) -> int | None:
142
142
  return self.batch_size
143
143
 
144
144
  @property
@@ -187,7 +187,7 @@ class CallableFunction(Function):
187
187
  return md, cloudpickle.dumps(self.py_fn)
188
188
 
189
189
  @classmethod
190
- def from_store(cls, name: Optional[str], md: dict, binary_obj: bytes) -> Function:
190
+ def from_store(cls, name: str | None, md: dict, binary_obj: bytes) -> Function:
191
191
  py_fn = cloudpickle.loads(binary_obj)
192
192
  assert callable(py_fn)
193
193
  sig = Signature.from_dict(md['signature'])
@@ -1,4 +1,4 @@
1
- from typing import Any, Optional, Sequence
1
+ from typing import Any, Sequence
2
2
 
3
3
  from pixeltable import exceptions as excs, exprs, type_system as ts
4
4
 
@@ -41,7 +41,7 @@ class ExprTemplateFunction(Function):
41
41
  templates: list[ExprTemplate]
42
42
  self_name: str
43
43
 
44
- def __init__(self, templates: list[ExprTemplate], self_path: Optional[str] = None, name: Optional[str] = None):
44
+ def __init__(self, templates: list[ExprTemplate], self_path: str | None = None, name: str | None = None):
45
45
  self.templates = templates
46
46
  self.self_name = name
47
47
 
@@ -85,29 +85,29 @@ class ExprTemplateFunction(Function):
85
85
  conditional_return_type).
86
86
  """
87
87
  assert not self.is_polymorphic
88
- template = self.template
89
88
  with_defaults = bound_args.copy()
90
89
  with_defaults.update(
91
- {param_name: default for param_name, default in template.defaults.items() if param_name not in bound_args}
90
+ {
91
+ param_name: default
92
+ for param_name, default in self.template.defaults.items()
93
+ if param_name not in bound_args
94
+ }
92
95
  )
93
96
  substituted_expr = self.template.expr.copy().substitute(
94
- {template.param_exprs[name]: expr for name, expr in with_defaults.items()}
97
+ {self.template.param_exprs[name]: expr for name, expr in with_defaults.items()}
95
98
  )
96
99
  return substituted_expr.col_type
97
100
 
98
- def comment(self) -> Optional[str]:
101
+ def comment(self) -> str | None:
99
102
  if isinstance(self.templates[0].expr, exprs.FunctionCall):
100
103
  return self.templates[0].expr.fn.comment()
101
104
  return None
102
105
 
103
106
  def exec(self, args: Sequence[Any], kwargs: dict[str, Any]) -> Any:
104
- from pixeltable import exec
105
-
106
107
  assert not self.is_polymorphic
107
108
  expr = self.instantiate(args, kwargs)
108
109
  row_builder = exprs.RowBuilder(output_exprs=[expr], columns=[], input_exprs=[])
109
- row_batch = exec.DataRowBatch(tbl=None, row_builder=row_builder, num_rows=1)
110
- row = row_batch[0]
110
+ row = row_builder.make_row()
111
111
  row_builder.eval(row, ctx=row_builder.default_eval_ctx)
112
112
  return row[row_builder.get_output_exprs()[0].slot_idx]
113
113
 
@@ -5,7 +5,7 @@ import inspect
5
5
  import typing
6
6
  from abc import ABC, abstractmethod
7
7
  from copy import copy
8
- from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence, cast
8
+ from typing import TYPE_CHECKING, Any, Callable, Sequence, cast
9
9
 
10
10
  import sqlalchemy as sql
11
11
  from typing_extensions import Self
@@ -30,10 +30,10 @@ class Function(ABC):
30
30
  """
31
31
 
32
32
  signatures: list[Signature]
33
- self_path: Optional[str]
33
+ self_path: str | None
34
34
  is_method: bool
35
35
  is_property: bool
36
- _conditional_return_type: Optional[Callable[..., ts.ColumnType]]
36
+ _conditional_return_type: Callable[..., ts.ColumnType] | None
37
37
 
38
38
  # We cache the overload resolutions in self._resolutions. This ensures that each resolution is represented
39
39
  # globally by a single Python object. We do this dynamically rather than pre-constructing them in order to
@@ -43,17 +43,17 @@ class Function(ABC):
43
43
  # Translates a call to this function with the given arguments to its SQLAlchemy equivalent.
44
44
  # Overriden for specific Function instances via the to_sql() decorator. The override must accept the same
45
45
  # parameter names as the original function. Each parameter is going to be of type sql.ColumnElement.
46
- _to_sql: Callable[..., Optional[sql.ColumnElement]]
46
+ _to_sql: Callable[..., sql.ColumnElement | None]
47
47
 
48
48
  # Returns the resource pool to use for calling this function with the given arguments.
49
49
  # Overriden for specific Function instances via the resource_pool() decorator. The override must accept a subset
50
50
  # of the parameters of the original function, with the same type.
51
- _resource_pool: Callable[..., Optional[str]]
51
+ _resource_pool: Callable[..., str | None]
52
52
 
53
53
  def __init__(
54
54
  self,
55
55
  signatures: list[Signature],
56
- self_path: Optional[str] = None,
56
+ self_path: str | None = None,
57
57
  is_method: bool = False,
58
58
  is_property: bool = False,
59
59
  ):
@@ -105,7 +105,7 @@ class Function(ABC):
105
105
  @abstractmethod
106
106
  def is_async(self) -> bool: ...
107
107
 
108
- def comment(self) -> Optional[str]:
108
+ def comment(self) -> str | None:
109
109
  return None
110
110
 
111
111
  def help_str(self) -> str:
@@ -175,7 +175,7 @@ class Function(ABC):
175
175
 
176
176
  def _bind_to_matching_signature(self, args: Sequence[Any], kwargs: dict[str, Any]) -> tuple[Self, dict[str, Any]]:
177
177
  result: int = -1
178
- bound_args: Optional[dict[str, Any]] = None
178
+ bound_args: dict[str, Any] | None = None
179
179
  assert len(self.signatures) > 0
180
180
  if len(self.signatures) == 1:
181
181
  # Only one signature: call _bind_to_signature() and surface any errors directly
@@ -206,7 +206,7 @@ class Function(ABC):
206
206
  self._resolved_fns[signature_idx].validate_call(normalized_args)
207
207
  return normalized_args
208
208
 
209
- def validate_call(self, bound_args: dict[str, Optional['exprs.Expr']]) -> None:
209
+ def validate_call(self, bound_args: dict[str, 'exprs.Expr' | None]) -> None:
210
210
  """Override this to do custom validation of the arguments"""
211
211
  assert not self.is_polymorphic
212
212
  self.signature.validate_args(bound_args, context=f'in function {self.name!r}')
@@ -252,9 +252,7 @@ class Function(ABC):
252
252
 
253
253
  return return_type
254
254
 
255
- def _assemble_callable_args(
256
- self, callable: Callable, bound_args: dict[str, 'exprs.Expr']
257
- ) -> Optional[dict[str, Any]]:
255
+ def _assemble_callable_args(self, callable: Callable, bound_args: dict[str, 'exprs.Expr']) -> dict[str, Any] | None:
258
256
  """
259
257
  Return the kwargs to pass to callable, given bound_args passed to this function.
260
258
 
@@ -286,7 +284,7 @@ class Function(ABC):
286
284
  return None
287
285
  assert isinstance(arg, exprs.Expr)
288
286
 
289
- expects_expr: Optional[type[exprs.Expr]] = None
287
+ expects_expr: type[exprs.Expr] | None = None
290
288
  type_hint = callable_type_hints.get(param.name)
291
289
  if typing.get_origin(type_hint) is not None:
292
290
  type_hint = typing.get_origin(type_hint) # Remove type subscript if one exists
@@ -410,12 +408,12 @@ class Function(ABC):
410
408
  """Execute the function with the given arguments and return the result."""
411
409
  raise NotImplementedError()
412
410
 
413
- def to_sql(self, fn: Callable[..., Optional[sql.ColumnElement]]) -> Callable[..., Optional[sql.ColumnElement]]:
411
+ def to_sql(self, fn: Callable[..., sql.ColumnElement | None]) -> Callable[..., sql.ColumnElement | None]:
414
412
  """Instance decorator for specifying the SQL translation of this function"""
415
413
  self._to_sql = fn
416
414
  return fn
417
415
 
418
- def __default_to_sql(self, *args: Any, **kwargs: Any) -> Optional[sql.ColumnElement]:
416
+ def __default_to_sql(self, *args: Any, **kwargs: Any) -> sql.ColumnElement | None:
419
417
  """The default implementation of SQL translation, which provides no translation"""
420
418
  return None
421
419
 
@@ -425,7 +423,7 @@ class Function(ABC):
425
423
  self._resource_pool = fn
426
424
  return fn
427
425
 
428
- def __default_resource_pool(self) -> Optional[str]:
426
+ def __default_resource_pool(self) -> str | None:
429
427
  return None
430
428
 
431
429
  def __eq__(self, other: object) -> bool:
@@ -495,7 +493,7 @@ class Function(ABC):
495
493
  raise NotImplementedError()
496
494
 
497
495
  @classmethod
498
- def from_store(cls, name: Optional[str], md: dict, binary_obj: bytes) -> Function:
496
+ def from_store(cls, name: str | None, md: dict, binary_obj: bytes) -> Function:
499
497
  """
500
498
  Create a Function instance from the serialized representation returned by to_store()
501
499
  """
@@ -504,12 +502,12 @@ class Function(ABC):
504
502
 
505
503
  class InvalidFunction(Function):
506
504
  fn_dict: dict[str, Any]
507
- errormsg: str
505
+ error_msg: str
508
506
 
509
- def __init__(self, self_path: str, fn_dict: dict[str, Any], errormsg: str):
507
+ def __init__(self, self_path: str, fn_dict: dict[str, Any], error_msg: str):
510
508
  super().__init__([], self_path)
511
509
  self.fn_dict = fn_dict
512
- self.errormsg = errormsg
510
+ self.error_msg = error_msg
513
511
 
514
512
  def _as_dict(self) -> dict:
515
513
  """
@@ -4,7 +4,6 @@ import dataclasses
4
4
  import importlib
5
5
  import logging
6
6
  import sys
7
- from typing import Optional
8
7
  from uuid import UUID
9
8
 
10
9
  import sqlalchemy as sql
@@ -23,7 +22,7 @@ class FunctionRegistry:
23
22
  Function are loaded from the store on demand.
24
23
  """
25
24
 
26
- _instance: Optional[FunctionRegistry] = None
25
+ _instance: FunctionRegistry | None = None
27
26
 
28
27
  @classmethod
29
28
  def get(cls) -> FunctionRegistry:
@@ -94,7 +93,7 @@ class FunctionRegistry:
94
93
  # stored_fn_md.append(md)
95
94
  return list(self.module_fns.values())
96
95
 
97
- # def get_function(self, *, id: Optional[UUID] = None, fqn: Optional[str] = None) -> Function:
96
+ # def get_function(self, *, id: UUID | None = None, fqn: str | None = None) -> Function:
98
97
  # assert (id is not None) != (fqn is not None)
99
98
  # if id is not None:
100
99
  # if id not in self.stored_fns_by_id:
@@ -143,7 +142,7 @@ class FunctionRegistry:
143
142
  return list(self.type_methods[base_type].values())
144
143
  return []
145
144
 
146
- def lookup_type_method(self, base_type: ts.ColumnType.Type, name: str) -> Optional[Function]:
145
+ def lookup_type_method(self, base_type: ts.ColumnType.Type, name: str) -> Function | None:
147
146
  """
148
147
  Look up a method (or property) by name for a given base type. If no such method is registered, return None.
149
148
  """
@@ -151,8 +150,8 @@ class FunctionRegistry:
151
150
  return self.type_methods[base_type][name]
152
151
  return None
153
152
 
154
- # def create_function(self, md: schema.FunctionMd, binary_obj: bytes, dir_id: Optional[UUID] = None) -> UUID:
155
- def create_stored_function(self, pxt_fn: Function, dir_id: Optional[UUID] = None) -> UUID:
153
+ # def create_function(self, md: schema.FunctionMd, binary_obj: bytes, dir_id: UUID | None = None) -> UUID:
154
+ def create_stored_function(self, pxt_fn: Function, dir_id: UUID | None = None) -> UUID:
156
155
  fn_md, binary_obj = pxt_fn.to_store()
157
156
  md = schema.FunctionMd(name=pxt_fn.name, md=fn_md, py_version=sys.version, class_name=pxt_fn.__class__.__name__)
158
157
  with env.Env.get().engine.begin() as conn:
@@ -184,7 +183,7 @@ class FunctionRegistry:
184
183
  self.stored_fns_by_id[id] = instance
185
184
  return instance
186
185
 
187
- # def create_function(self, fn: Function, dir_id: Optional[UUID] = None, name: Optional[str] = None) -> None:
186
+ # def create_function(self, fn: Function, dir_id: UUID | None = None, name: str | None = None) -> None:
188
187
  # with env.Env.get().engine.begin() as conn:
189
188
  # _logger.debug(f'Pickling function {name}')
190
189
  # eval_fn_str = cloudpickle.dumps(fn.eval_fn) if fn.eval_fn is not None else None
@@ -1,14 +1,13 @@
1
1
  import importlib
2
2
  import inspect
3
3
  from types import ModuleType
4
- from typing import Optional
5
4
 
6
5
  import pixeltable.exceptions as excs
7
6
 
8
7
 
9
- def resolve_symbol(symbol_path: str) -> Optional[object]:
8
+ def resolve_symbol(symbol_path: str) -> object | None:
10
9
  path_elems = symbol_path.split('.')
11
- module: Optional[ModuleType] = None
10
+ module: ModuleType | None = None
12
11
  i = len(path_elems) - 1
13
12
  while i > 0 and module is None:
14
13
  try:
pixeltable/func/mcp.py ADDED
@@ -0,0 +1,74 @@
1
+ import asyncio
2
+ import inspect
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ import pixeltable as pxt
6
+ from pixeltable import exceptions as excs, type_system as ts
7
+ from pixeltable.func.signature import Parameter
8
+
9
+ if TYPE_CHECKING:
10
+ import mcp
11
+
12
+
13
+ def mcp_udfs(url: str) -> list['pxt.func.Function']:
14
+ return asyncio.run(mcp_udfs_async(url))
15
+
16
+
17
+ async def mcp_udfs_async(url: str) -> list['pxt.func.Function']:
18
+ import mcp
19
+ from mcp.client.streamable_http import streamablehttp_client
20
+
21
+ list_tools_result: mcp.types.ListToolsResult | None = None
22
+ async with (
23
+ streamablehttp_client(url) as (read_stream, write_stream, _),
24
+ mcp.ClientSession(read_stream, write_stream) as session,
25
+ ):
26
+ await session.initialize()
27
+ list_tools_result = await session.list_tools()
28
+ assert list_tools_result is not None
29
+
30
+ return [mcp_tool_to_udf(url, tool) for tool in list_tools_result.tools]
31
+
32
+
33
+ def mcp_tool_to_udf(url: str, mcp_tool: 'mcp.types.Tool') -> 'pxt.func.Function':
34
+ import mcp
35
+ from mcp.client.streamable_http import streamablehttp_client
36
+
37
+ async def invoke(**kwargs: Any) -> str:
38
+ # TODO: Cache session objects rather than creating a new one each time?
39
+ async with (
40
+ streamablehttp_client(url) as (read_stream, write_stream, _),
41
+ mcp.ClientSession(read_stream, write_stream) as session,
42
+ ):
43
+ await session.initialize()
44
+ res = await session.call_tool(name=mcp_tool.name, arguments=kwargs)
45
+ # TODO Handle image/audio responses?
46
+ return res.content[0].text # type: ignore[union-attr]
47
+
48
+ if mcp_tool.description is not None:
49
+ invoke.__doc__ = mcp_tool.description
50
+
51
+ input_schema = mcp_tool.inputSchema
52
+ params = {
53
+ name: __mcp_param_to_pxt_type(mcp_tool.name, name, param) for name, param in input_schema['properties'].items()
54
+ }
55
+ required = input_schema.get('required', [])
56
+
57
+ # Ensure that any params not appearing in `required` are nullable.
58
+ # (A required param might or might not be nullable, since its type might be an 'anyOf' containing a null.)
59
+ for name in params.keys() - required:
60
+ params[name] = params[name].copy(nullable=True)
61
+
62
+ signature = pxt.func.Signature(
63
+ return_type=ts.StringType(), # Return type is always string
64
+ parameters=[Parameter(name, col_type, inspect.Parameter.KEYWORD_ONLY) for name, col_type in params.items()],
65
+ )
66
+
67
+ return pxt.func.CallableFunction(signatures=[signature], py_fns=[invoke], self_name=mcp_tool.name)
68
+
69
+
70
+ def __mcp_param_to_pxt_type(tool_name: str, name: str, param: dict[str, Any]) -> ts.ColumnType:
71
+ pxt_type = ts.ColumnType.from_json_schema(param)
72
+ if pxt_type is None:
73
+ raise excs.Error(f'Unknown type schema for MCP parameter {name!r} of tool {tool_name!r}: {param}')
74
+ return pxt_type
@@ -2,7 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import inspect
4
4
  from functools import reduce
5
- from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional, Union, overload
5
+ from typing import TYPE_CHECKING, Any, Callable, Iterable, overload
6
6
 
7
7
  from pixeltable import catalog, exceptions as excs, exprs, func, type_system as ts
8
8
 
@@ -16,13 +16,13 @@ if TYPE_CHECKING:
16
16
  class QueryTemplateFunction(Function):
17
17
  """A parameterized query/DataFrame from which an executable DataFrame is created with a function call."""
18
18
 
19
- template_df: Optional['DataFrame']
20
- self_name: Optional[str]
21
- _comment: Optional[str]
19
+ template_df: 'DataFrame' | None
20
+ self_name: str | None
21
+ _comment: str | None
22
22
 
23
23
  @classmethod
24
24
  def create(
25
- cls, template_callable: Callable, param_types: Optional[list[ts.ColumnType]], path: str, name: str
25
+ cls, template_callable: Callable, param_types: list[ts.ColumnType] | None, path: str, name: str
26
26
  ) -> QueryTemplateFunction:
27
27
  # we need to construct a template df and a signature
28
28
  py_sig = inspect.signature(template_callable)
@@ -40,11 +40,11 @@ class QueryTemplateFunction(Function):
40
40
 
41
41
  def __init__(
42
42
  self,
43
- template_df: Optional['DataFrame'],
43
+ template_df: 'DataFrame' | None,
44
44
  sig: Signature,
45
- path: Optional[str] = None,
46
- name: Optional[str] = None,
47
- comment: Optional[str] = None,
45
+ path: str | None = None,
46
+ name: str | None = None,
47
+ comment: str | None = None,
48
48
  ):
49
49
  assert sig is not None
50
50
  super().__init__([sig], self_path=path)
@@ -82,7 +82,7 @@ class QueryTemplateFunction(Function):
82
82
  def name(self) -> str:
83
83
  return self.self_name
84
84
 
85
- def comment(self) -> Optional[str]:
85
+ def comment(self) -> str | None:
86
86
  return self._comment
87
87
 
88
88
  def _as_dict(self) -> dict:
@@ -100,11 +100,11 @@ def query(py_fn: Callable) -> QueryTemplateFunction: ...
100
100
 
101
101
 
102
102
  @overload
103
- def query(*, param_types: Optional[list[ts.ColumnType]] = None) -> Callable[[Callable], QueryTemplateFunction]: ...
103
+ def query(*, param_types: list[ts.ColumnType] | None = None) -> Callable[[Callable], QueryTemplateFunction]: ...
104
104
 
105
105
 
106
106
  def query(*args: Any, **kwargs: Any) -> Any:
107
- def make_query_template(py_fn: Callable, param_types: Optional[list[ts.ColumnType]]) -> QueryTemplateFunction:
107
+ def make_query_template(py_fn: Callable, param_types: list[ts.ColumnType] | None) -> QueryTemplateFunction:
108
108
  if py_fn.__module__ != '__main__' and py_fn.__name__.isidentifier():
109
109
  # this is a named function in a module
110
110
  function_path = f'{py_fn.__module__}.{py_fn.__qualname__}'
@@ -127,10 +127,10 @@ def query(*args: Any, **kwargs: Any) -> Any:
127
127
 
128
128
  def retrieval_udf(
129
129
  table: catalog.Table,
130
- name: Optional[str] = None,
131
- description: Optional[str] = None,
132
- parameters: Optional[Iterable[Union[str, exprs.ColumnRef]]] = None,
133
- limit: Optional[int] = 10,
130
+ name: str | None = None,
131
+ description: str | None = None,
132
+ parameters: Iterable[str | exprs.ColumnRef] | None = None,
133
+ limit: int | None = 10,
134
134
  ) -> func.QueryTemplateFunction:
135
135
  """
136
136
  Constructs a retrieval UDF for the given table. The retrieval UDF is a UDF whose parameters are
@@ -157,11 +157,13 @@ def retrieval_udf(
157
157
  """
158
158
  # Argument validation
159
159
  col_refs: list[exprs.ColumnRef]
160
+ # TODO: get rid of references to ColumnRef internals and replace instead with a public interface
161
+ col_names = table.columns()
160
162
  if parameters is None:
161
- col_refs = [table[col_name] for col_name in table.columns if not table[col_name].col.is_computed]
163
+ col_refs = [table[col_name] for col_name in col_names if not table[col_name].col.is_computed]
162
164
  else:
163
165
  for param in parameters:
164
- if isinstance(param, str) and param not in table.columns:
166
+ if isinstance(param, str) and param not in col_names:
165
167
  raise excs.Error(f'The specified parameter {param!r} is not a column of the table {table._path()!r}')
166
168
  col_refs = [table[param] if isinstance(param, str) else param for param in parameters]
167
169