pixeltable 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

pixeltable/func/function_registry.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import dataclasses
+import importlib
+import logging
+import sys
+import types
+from typing import Optional, Dict, List, Tuple
+from uuid import UUID
+
+import cloudpickle
+import sqlalchemy as sql
+
+import pixeltable.env as env
+import pixeltable.exceptions as excs
+import pixeltable.type_system as ts
+from pixeltable.metadata import schema
+from .function import Function
+from .globals import get_caller_module_path
+
+_logger = logging.getLogger('pixeltable')
+
+class FunctionRegistry:
+    """
+    A central registry for all Functions. Handles interactions with the backing store.
+    Function are loaded from the store on demand.
+    """
+    _instance: Optional[FunctionRegistry] = None
+
+    @classmethod
+    def get(cls) -> FunctionRegistry:
+        if cls._instance is None:
+            cls._instance = FunctionRegistry()
+        return cls._instance
+
+    def __init__(self):
+        self.stored_fns_by_id: Dict[UUID, Function] = {}
+        self.module_fns: Dict[str, Function] = {}  # fqn -> Function
+
+    def clear_cache(self) -> None:
+        """
+        Useful during testing
+        """
+        self.stored_fns_by_id: Dict[UUID, Function] = {}
+
+    # def register_std_modules(self) -> None:
+    #     """Register all submodules of pixeltable.functions"""
+    #     root = sys.modules['pixeltable.functions']
+    #     self.register_submodules(root)
+    #
+    # def register_submodules(self, mod: types.ModuleType) -> None:
+    #     # TODO: this doesn't work
+    #     for name, submod in mod.__dict__.items():
+    #         if isinstance(submod, types.ModuleType):
+    #             self.register_module(submod)
+    #             self.register_submodules(submod)
+    #
+    # def register_module(self) -> None:
+    #     """Register all Functions in the caller module"""
+    #     caller_path = get_caller_module_path()
+    #     mod = importlib.import_module(caller_path)
+    #     for name in dir(mod):
+    #         obj = getattr(mod, name)
+    #         if isinstance(obj, Function):
+    #             fn_path = f'{caller_path}.{name}'  # fully-qualified name
+    #             self.module_fns[fn_path] = obj
+
+    def register_function(self, fqn: str, fn: Function) -> None:
+        self.module_fns[fqn] = fn
+
+    def list_functions(self) -> List[Function]:
+        # retrieve Function.Metadata data for all existing stored functions from store directly
+        # (self.stored_fns_by_id isn't guaranteed to contain all functions)
+        # TODO: have the client do this, once the client takes over the Db functionality
+        # stmt = sql.select(
+        #         schema.Function.name, schema.Function.md,
+        #         schema.Db.name, schema.Dir.path, sql_func.length(schema.Function.init_obj))\
+        #     .where(schema.Function.db_id == schema.Db.id)\
+        #     .where(schema.Function.dir_id == schema.Dir.id)
+        # stored_fn_md: List[Function.Metadata] = []
+        # with Env.get().engine.begin() as conn:
+        #     rows = conn.execute(stmt)
+        #     for name, md_dict, db_name, dir_path, init_obj_len in rows:
+        #         md = Function.Metadata.from_dict(md_dict)
+        #         md.fqn = f'{db_name}{"." + dir_path if dir_path != "" else ""}.{name}'
+        #         stored_fn_md.append(md)
+        return list(self.module_fns.values())
+
+    # def get_function(self, *, id: Optional[UUID] = None, fqn: Optional[str] = None) -> Function:
+    #     assert (id is not None) != (fqn is not None)
+    #     if id is not None:
+    #         if id not in self.stored_fns_by_id:
+    #             stmt = sql.select(
+    #                     schema.Function.md, schema.Function.eval_obj, schema.Function.init_obj,
+    #                     schema.Function.update_obj, schema.Function.value_obj) \
+    #                 .where(schema.Function.id == id)
+    #             with env.Env.get().engine.begin() as conn:
+    #                 rows = conn.execute(stmt)
+    #                 row = next(rows)
+    #                 schema_md = schema.md_from_dict(schema.FunctionMd, row[0])
+    #                 name = schema_md.name
+    #                 md = FunctionMd.from_dict(schema_md.md)
+    #                 # md.fqn is set by caller
+    #                 eval_fn = cloudpickle.loads(row[1]) if row[1] is not None else None
+    #                 # TODO: are these checks needed?
+    #                 if row[1] is not None and eval_fn is None:
+    #                     raise excs.Error(f'Could not load eval_fn for function {name}')
+    #                 init_fn = cloudpickle.loads(row[2]) if row[2] is not None else None
+    #                 if row[2] is not None and init_fn is None:
+    #                     raise excs.Error(f'Could not load init_fn for aggregate function {name}')
+    #                 update_fn = cloudpickle.loads(row[3]) if row[3] is not None else None
+    #                 if row[3] is not None and update_fn is None:
+    #                     raise excs.Error(f'Could not load update_fn for aggregate function {name}')
+    #                 value_fn = cloudpickle.loads(row[4]) if row[4] is not None else None
+    #                 if row[4] is not None and value_fn is None:
+    #                     raise excs.Error(f'Could not load value_fn for aggregate function {name}')
+    #
+    #                 func = Function(
+    #                     md, id=id,
+    #                     eval_fn=eval_fn, init_fn=init_fn, update_fn=update_fn, value_fn=value_fn)
+    #                 _logger.info(f'Loaded function {name} from store')
+    #                 self.stored_fns_by_id[id] = func
+    #         assert id in self.stored_fns_by_id
+    #         return self.stored_fns_by_id[id]
+    #     else:
+    #         # this is an already-registered library function
+    #         assert fqn in self.module_fns, f'{fqn} not found'
+    #         return self.module_fns[fqn]
+
+    def get_type_methods(self, name: str, base_type: ts.ColumnType.Type) -> List[Function]:
+        return [
+            fn for fn in self.module_fns.values()
+            if fn.self_path is not None and fn.self_path.endswith('.' + name) \
+               and fn.signature.parameters_by_pos[0].col_type.type_enum == base_type
+        ]
+
+    #def create_function(self, md: schema.FunctionMd, binary_obj: bytes, dir_id: Optional[UUID] = None) -> UUID:
+    def create_stored_function(self, pxt_fn: Function, dir_id: Optional[UUID] = None) -> UUID:
+        fn_md, binary_obj = pxt_fn.to_store()
+        md = schema.FunctionMd(name=pxt_fn.name, md=fn_md, py_version=sys.version, class_name=pxt_fn.__class__.__name__)
+        with env.Env.get().engine.begin() as conn:
+            res = conn.execute(
+                sql.insert(schema.Function.__table__)
+                    .values(dir_id=dir_id, md=dataclasses.asdict(md), binary_obj=binary_obj))
+            id = res.inserted_primary_key[0]
+            _logger.info(f'Created function {pxt_fn.name} (id {id}) in store')
+            self.stored_fns_by_id[id] = pxt_fn
+            return id
+
+    def get_stored_function(self, id: UUID) -> Function:
+        if id in self.stored_fns_by_id:
+            return self.stored_fns_by_id[id]
+        stmt = sql.select(schema.Function.md, schema.Function.binary_obj, schema.Function.dir_id)\
+            .where(schema.Function.id == id)
+        with env.Env.get().engine.begin() as conn:
+            row = conn.execute(stmt).fetchone()
+            if row is None:
+                raise excs.Error(f'Function with id {id} not found')
+            # create instance of the referenced class
+            md = schema.md_from_dict(schema.FunctionMd, row[0])
+            func_module = importlib.import_module(self.__module__.rsplit('.', 1)[0])
+            func_class = getattr(func_module, md.class_name)
+            instance = func_class.from_store(md.name, md.md, row[1])
+            self.stored_fns_by_id[id] = instance
+            return instance
+
+    # def create_function(self, fn: Function, dir_id: Optional[UUID] = None, name: Optional[str] = None) -> None:
+    #     with env.Env.get().engine.begin() as conn:
+    #         _logger.debug(f'Pickling function {name}')
+    #         eval_fn_str = cloudpickle.dumps(fn.eval_fn) if fn.eval_fn is not None else None
+    #         init_fn_str = cloudpickle.dumps(fn.init_fn) if fn.init_fn is not None else None
+    #         update_fn_str = cloudpickle.dumps(fn.update_fn) if fn.update_fn is not None else None
+    #         value_fn_str = cloudpickle.dumps(fn.value_fn) if fn.value_fn is not None else None
+    #         total_size = \
+    #             (len(eval_fn_str) if eval_fn_str is not None else 0) + \
+    #             (len(init_fn_str) if init_fn_str is not None else 0) + \
+    #             (len(update_fn_str) if update_fn_str is not None else 0) + \
+    #             (len(value_fn_str) if value_fn_str is not None else 0)
+    #         _logger.debug(f'Pickled function {name} ({total_size} bytes)')
+    #
+    #         schema_md = schema.FunctionMd(name=name, md=fn.md.as_dict())
+    #         res = conn.execute(
+    #             sql.insert(schema.Function.__table__)
+    #                 .values(
+    #                     dir_id=dir_id, md=dataclasses.asdict(schema_md),
+    #                     eval_obj=eval_fn_str, init_obj=init_fn_str, update_obj=update_fn_str, value_obj=value_fn_str))
+    #         fn.id = res.inserted_primary_key[0]
+    #         self.stored_fns_by_id[fn.id] = fn
+    #         _logger.info(f'Created function {name} in store')
+
+    # def update_function(self, id: UUID, new_fn: Function) -> None:
+    #     """
+    #     Updates the callables for the Function with the given id in the store and in the cache, if present.
+    #     """
+    #     assert not new_fn.is_module_function
+    #     with env.Env.get().engine.begin() as conn:
+    #         updates = {}
+    #         if new_fn.eval_fn is not None:
+    #             updates[schema.Function.eval_obj] = cloudpickle.dumps(new_fn.eval_fn)
+    #         if new_fn.init_fn is not None:
+    #             updates[schema.Function.init_obj] = cloudpickle.dumps(new_fn.init_fn)
+    #         if new_fn.update_fn is not None:
+    #             updates[schema.Function.update_obj] = cloudpickle.dumps(new_fn.update_fn)
+    #         if new_fn.value_fn is not None:
+    #             updates[schema.Function.value_obj] = cloudpickle.dumps(new_fn.value_fn)
+    #         conn.execute(
+    #             sql.update(schema.Function.__table__)
+    #                 .values(updates)
+    #                 .where(schema.Function.id == id))
+    #         _logger.info(f'Updated function {new_fn.md.fqn} (id={id}) in store')
+    #     if id in self.stored_fns_by_id:
+    #         if new_fn.eval_fn is not None:
+    #             self.stored_fns_by_id[id].eval_fn = new_fn.eval_fn
+    #         if new_fn.init_fn is not None:
+    #             self.stored_fns_by_id[id].init_fn = new_fn.init_fn
+    #         if new_fn.update_fn is not None:
+    #             self.stored_fns_by_id[id].update_fn = new_fn.update_fn
+    #         if new_fn.value_fn is not None:
+    #             self.stored_fns_by_id[id].value_fn = new_fn.value_fn
+
+    def delete_function(self, id: UUID) -> None:
+        assert id is not None
+        with env.Env.get().engine.begin() as conn:
+            conn.execute(
+                sql.delete(schema.Function.__table__)
+                    .where(schema.Function.id == id))
+            _logger.info(f'Deleted function with id {id} from store')

pixeltable/func/globals.py

@@ -0,0 +1,36 @@
+from typing import Optional
+from types import ModuleType
+import importlib
+import inspect
+
+
+def resolve_symbol(symbol_path: str) -> object:
+    path_elems = symbol_path.split('.')
+    module: Optional[ModuleType] = None
+    if path_elems[0:2] == ['pixeltable', 'functions'] and len(path_elems) > 2:
+        # if this is a pixeltable.functions submodule, it cannot be resolved via pixeltable.functions;
+        # try to import the submodule directly
+        submodule_path = '.'.join(path_elems[0:3])
+        try:
+            module = importlib.import_module(submodule_path)
+            path_elems = path_elems[3:]
+        except ModuleNotFoundError:
+            pass
+    if module is None:
+        module = importlib.import_module(path_elems[0])
+        path_elems = path_elems[1:]
+    obj = module
+    for el in path_elems:
+        obj = getattr(obj, el)
+    return obj
+
+def get_caller_module_path() -> str:
+    """Return the module path of our caller's caller"""
+    stack = inspect.stack()
+    try:
+        caller_frame = stack[2].frame
+        module_path = caller_frame.f_globals['__name__']
+    finally:
+        # remove references to stack frames to avoid reference cycles
+        del stack
+    return module_path

pixeltable/func/nos_function.py

@@ -0,0 +1,202 @@
+from typing import Optional, Any, Dict, List, Tuple
+import inspect
+import logging
+import sys
+
+import numpy as np
+
+from .signature import Signature, Parameter
+from .batched_function import BatchedFunction
+import pixeltable.env as env
+import pixeltable.type_system as ts
+import pixeltable.exceptions as excs
+
+
+_logger = logging.getLogger('pixeltable')
+
+class NOSFunction(BatchedFunction):
+    def __init__(self, model_spec: 'nos.common.ModelSpec', self_path: str):
+        return_type, param_types = self._convert_nos_signature(model_spec.signature)
+        param_names = list(model_spec.signature.get_inputs_spec().keys())
+        params = [
+            Parameter(name, col_type, inspect.Parameter.POSITIONAL_OR_KEYWORD, is_batched=False)
+            for name, col_type in zip(param_names, param_types)
+        ]
+        signature = Signature(return_type, params)
+
+        # construct inspect.Signature
+        py_params = [
+            inspect.Parameter(name, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+            for name, col_type in zip(param_names, param_types)
+        ]
+        py_signature = inspect.Signature(py_params)
+        super().__init__(signature, py_signature=py_signature, self_path=self_path)
+
+        self.model_spec = model_spec
+        self.nos_param_names = model_spec.signature.get_inputs_spec().keys()
+        self.scalar_nos_param_names = []
+
+        # for models on images
+        self.img_param_pos: Optional[int] = None  # position of the image parameter in the function signature
+        # for multi-resolution image models
+        import nos
+        self.img_batch_params: List[nos.common.ObjectTypeInfo] = []
+        self.img_resolutions: List[int] = []  # for multi-resolution models
+        self.batch_size: Optional[int] = None
+        self.img_size: Optional[Tuple[int, int]] = None  # W, H
+
+        # try to determine batch_size and img_size
+        batch_size = sys.maxsize
+        for pos, (param_name, type_info) in enumerate(model_spec.signature.get_inputs_spec().items()):
+            if isinstance(type_info, list):
+                assert isinstance(type_info[0].base_spec(), nos.common.ImageSpec)
+                # this is a multi-resolution image model
+                self.img_batch_params = type_info
+                self.img_param_pos = pos
+                self.img_resolutions = [
+                    info.base_spec().shape[0] * info.base_spec().shape[1] for info in self.img_batch_params
+                ]
+            else:
+                if not type_info.is_batched():
+                    self.scalar_nos_param_names.append(param_name)
+                else:
+                    batch_size = min(batch_size, type_info.batch_size())
+
+                if isinstance(type_info.base_spec(), nos.common.ImageSpec):
+                    # this is a single-resolution image model
+                    if type_info.base_spec().shape is not None:
+                        self.img_size = (type_info.base_spec().shape[1], type_info.base_spec().shape[0])
+                    self.img_param_pos = pos
+
+        if batch_size != sys.maxsize:
+            self.batch_size = batch_size
+
+    def _convert_nos_type(
+            self, type_info: 'nos.common.spec.ObjectTypeInfo', ignore_shape: bool = False
+    ) -> ts.ColumnType:
+        """Convert ObjectTypeInfo to ColumnType"""
+        import nos
+        if type_info.base_spec() is None:
+            if type_info.base_type() == str:
+                return ts.StringType()
+            if type_info.base_type() == int:
+                return ts.IntType()
+            if type_info.base_type() == float:
+                return ts.FloatType()
+            if type_info.base_type() == bool:
+                return ts.BoolType()
+            else:
+                raise excs.Error(f'Cannot convert {type_info} to ColumnType')
+        elif isinstance(type_info.base_spec(), nos.common.ImageSpec):
+            size = None
+            if not ignore_shape and type_info.base_spec().shape is not None:
+                size = (type_info.base_spec().shape[1], type_info.base_spec().shape[0])
+            # TODO: set mode
+            return ts.ImageType(size=size)
+        elif isinstance(type_info.base_spec(), nos.common.TensorSpec):
+            return ts.ArrayType(shape=type_info.base_spec().shape, dtype=ts.FloatType())
+        else:
+            raise excs.Error(f'Cannot convert {type_info} to ColumnType')
+
+    def _convert_nos_signature(
+            self, sig: 'nos.common.spec.FunctionSignature') -> Tuple[ts.ColumnType, List[ts.ColumnType]]:
+        if len(sig.get_outputs_spec()) > 1:
+            return_type = ts.JsonType()
+        else:
+            return_type = self._convert_nos_type(list(sig.get_outputs_spec().values())[0])
+        param_types: List[ts.ColumnType] = []
+        for _, type_info in sig.get_inputs_spec().items():
+            # if there are multiple input shapes we leave them out of the ColumnType and deal with them in FunctionCall
+            if isinstance(type_info, list):
+                param_types.append(self._convert_nos_type(type_info[0], ignore_shape=True))
+            else:
+                param_types.append(self._convert_nos_type(type_info, ignore_shape=False))
+        return return_type, param_types
+
+    def is_multi_res_model(self) -> bool:
+        return self.img_param_pos is not None and len(self.img_batch_params) > 0
+
+    def get_batch_size(self, *args: Any, **kwargs: Any) -> Optional[int]:
+        if self.batch_size is not None or len(self.img_batch_params) == 0 or len(args) == 0:
+            return self.batch_size
+
+        # return batch size appropriate for the given image size
+        img_arg = args[self.img_param_pos]
+        input_res = img_arg.size[0] * img_arg.size[1]
+        batch_size, _ = self._select_model_res(input_res)
+        return batch_size
+
+    def _select_model_res(self, input_res: int) -> Tuple[int, Tuple[int, int]]:
+        """Select the model resolution that is closest to the input resolution
+        Returns: batch size, image size
+        """
+        deltas = [abs(res - input_res) for res in self.img_resolutions]
+        idx = deltas.index(min(deltas))
+        type_info = self.img_batch_params[idx]
+        return type_info.batch_size(), (type_info.base_spec().shape[1], type_info.base_spec().shape[0])
+
+    def invoke(self, arg_batches: List[List[Any]], kwarg_batches: Dict[str, List[Any]]) -> List[Any]:
+        # check that scalar args are constant
+
+        num_batch_rows = len(arg_batches[0])
+        # if we need to rescale image args, and we're doing object detection, we need to rescale the
+        # bounding boxes as well
+        scale_factors = np.ndarray((num_batch_rows, 2), dtype=np.float32)
+
+        target_res: Optional[Tuple[int, int]] = None
+        if self.img_param_pos is not None:
+            # for now, NOS will only receive RGB images
+            arg_batches[self.img_param_pos] = \
+                [img.convert('RGB') if img.mode != 'RGB' else img for img in arg_batches[self.img_param_pos ]]
+            if self.is_multi_res_model():
+                # we need to select the resolution that is closest to the input resolution
+                sample_img = arg_batches[self.img_param_pos][0]
+                _, target_res = self._select_model_res(sample_img.size[0] * sample_img.size[1])
+            else:
+                target_res = self.img_size
+
+            if target_res is not None:
+                # we need to record the scale factors and resize the images;
+                # keep in mind that every image could have a different resolution
+                scale_factors[:, 0] = \
+                    [img.size[0] / target_res[0] for img in arg_batches[self.img_param_pos]]
+                scale_factors[:, 1] = \
+                    [img.size[1] / target_res[1] for img in arg_batches[self.img_param_pos]]
+                arg_batches[self.img_param_pos] = [
+                    # only resize if necessary
+                    img.resize(target_res) if img.size != target_res else img
+                    for img in arg_batches[self.img_param_pos]
+                ]
+
+        kwargs = {param_name: args for param_name, args in zip(self.nos_param_names, arg_batches)}
+        # fix up scalar parameters
+        kwargs.update(
+            {param_name: kwargs[param_name][0] for param_name in self.scalar_nos_param_names})
+        _logger.debug(
+            f'Running NOS task {self.model_spec.task}: '
+            f'batch_size={num_batch_rows} target_res={target_res}')
+        result = env.Env.get().nos_client.Run(
+            task=self.model_spec.task, model_name=self.model_spec.name, **kwargs)
+
+        import nos
+        if self.model_spec.task == nos.common.TaskType.OBJECT_DETECTION_2D and target_res is not None:
+            # we need to rescale the bounding boxes
+            result_bboxes = []  # workaround: result['bboxes'][*] is immutable
+            for i, bboxes in enumerate(result['bboxes']):
+                bboxes = np.copy(bboxes)
+                nos_batch_row_idx = i
+                bboxes[:, 0] *= scale_factors[nos_batch_row_idx, 0]
+                bboxes[:, 1] *= scale_factors[nos_batch_row_idx, 1]
+                bboxes[:, 2] *= scale_factors[nos_batch_row_idx, 0]
+                bboxes[:, 3] *= scale_factors[nos_batch_row_idx, 1]
+                result_bboxes.append(bboxes)
+            result['bboxes'] = result_bboxes
+
+        if len(result) == 1:
+            key = list(result.keys())[0]
+            row_results = result[key]
+        else:
+            # we rearrange result into one dict per row
+            row_results = [{k: v[i].tolist() for k, v in result.items()} for i in range(num_batch_rows)]
+        return row_results
+

pixeltable/func/signature.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import dataclasses
+import enum
+import inspect
+import logging
+import typing
+from typing import Optional, Callable, Dict, List, Any, Union, Tuple
+
+import pixeltable.exceptions as excs
+import pixeltable.type_system as ts
+
+_logger = logging.getLogger('pixeltable')
+
+
+@dataclasses.dataclass
+class Parameter:
+    name: str
+    col_type: Optional[ts.ColumnType]  # None for variable parameters
+    kind: enum.Enum  # inspect.Parameter.kind; inspect._ParameterKind is private
+    is_batched: bool = False  # True if the parameter is a batched parameter (eg, Batch[dict])
+
+
+T = typing.TypeVar('T')
+Batch = typing.Annotated[list[T], 'pxt-batch']
+
+
+class Signature:
+    """
+    Represents the signature of a Pixeltable function.
+
+    Regarding return type:
+    - most functions will have a fixed return type, which is specified directly
+    - some functions will have a return type that depends on the argument values;
+      ex.: PIL.Image.Image.resize() returns an image with dimensions specified as a parameter
+    - in the latter case, the 'return_type' field is a function that takes the bound arguments and returns the
+      return type; if no bound arguments are specified, a generic return type is returned (eg, ImageType() without a
+      size)
+    - self.is_batched: return type is a Batch[...] type
+    """
+    SPECIAL_PARAM_NAMES = ['group_by', 'order_by']
+
+    def __init__(
+            self,
+            return_type: Union[ts.ColumnType, Callable[[Dict[str, Any]], ts.ColumnType]],
+            parameters: List[Parameter], is_batched: bool = False):
+        self.return_type = return_type
+        self.is_batched = is_batched
+        # we rely on the ordering guarantee of dicts in Python >=3.7
+        self.parameters = {p.name: p for p in parameters}
+        self.parameters_by_pos = parameters.copy()
+        self.constant_parameters = [p for p in parameters if not p.is_batched]
+        self.batched_parameters = [p for p in parameters if p.is_batched]
+
+    def get_return_type(self, bound_args: Optional[Dict[str, Any]] = None) -> ts.ColumnType:
+        if isinstance(self.return_type, ts.ColumnType):
+            return self.return_type
+        return self.return_type(bound_args)
+
+    def as_dict(self) -> Dict[str, Any]:
+        result = {
+            'return_type': self.get_return_type().as_dict(),
+            'parameters': [
+                [p.name, p.col_type.as_dict() if p.col_type is not None else None, p.kind, p.is_batched]
+                for p in self.parameters.values()
+            ]
+        }
+        return result
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> Signature:
+        parameters = [Parameter(p[0], ts.ColumnType.from_dict(p[1]), p[2], p[3]) for p in d['parameters']]
+        return cls(ts.ColumnType.from_dict(d['return_type']), parameters)
+
+    def __eq__(self, other: Signature) -> bool:
+        if self.get_return_type() != other.get_return_type():
+            return False
+        if len(self.parameters) != len(other.parameters):
+            return False
+        # ignore the parameter name
+        for param, other_param in zip(self.parameters.values(), other.parameters.values()):
+            if param.col_type != other_param.col_type or param.kind != other_param.kind:
+                return False
+        return True
+
+    def __str__(self) -> str:
+        param_strs: List[str] = []
+        for p in self.parameters.values():
+            if p.kind == inspect.Parameter.VAR_POSITIONAL:
+                param_strs.append(f'*{p.name}')
+            elif p.kind == inspect.Parameter.VAR_KEYWORD:
+                param_strs.append(f'**{p.name}')
+            else:
+                param_strs.append(f'{p.name}: {str(p.col_type)}')
+        return f'({", ".join(param_strs)}) -> {str(self.get_return_type())}'
+
+    @classmethod
+    def _infer_type(cls, annotation: Optional[type]) -> Tuple[Optional[ts.ColumnType], Optional[bool]]:
+        """Returns: (column type, is_batched) or (None, ...) if the type cannot be inferred"""
+        if annotation is None:
+            return (None, None)
+        py_type: Optional[type] = None
+        is_batched = False
+        if typing.get_origin(annotation) == typing.Annotated:
+            type_args = typing.get_args(annotation)
+            if len(type_args) == 2 and type_args[1] == 'pxt-batch':
+                # this is our Batch
+                assert typing.get_origin(type_args[0]) == list
+                is_batched = True
+                py_type = typing.get_args(type_args[0])[0]
+        if py_type is None:
+            py_type = annotation
+        col_type = ts.ColumnType.from_python_type(py_type)
+        return (col_type, is_batched)
+
+    @classmethod
+    def create(
+            cls, c: Callable,
+            param_types: Optional[List[ts.ColumnType]] = None,
+            return_type: Optional[Union[ts.ColumnType, Callable]] = None
+    ) -> Signature:
+        """Create a signature for the given Callable.
+        Infer the parameter and return types, if none are specified.
+        Raises an exception if the types cannot be inferred.
+        """
+        sig = inspect.signature(c)
+        py_parameters = list(sig.parameters.values())
+
+        # check non-var parameters for name collisions and default value compatibility
+        parameters: List[Parameter] = []
+        for idx, param in enumerate(py_parameters):
+            if param.name in cls.SPECIAL_PARAM_NAMES:
+                raise excs.Error(f"'{param.name}' is a reserved parameter name")
+            if param.kind == inspect.Parameter.VAR_POSITIONAL or param.kind == inspect.Parameter.VAR_KEYWORD:
+                parameters.append(Parameter(param.name, None, param.kind, False))
+                continue
+
+            if param_types is not None:
+                if idx >= len(param_types):
+                    raise excs.Error(f'Missing type for parameter {param.name}')
+                param_type = param_types[idx]
+                is_batched = False
+            else:
+                param_type, is_batched = cls._infer_type(param.annotation)
+                if param_type is None:
+                    raise excs.Error(f'Cannot infer pixeltable type for parameter {param.name}')
+
+            # check default value compatibility
+            default_val = sig.parameters[param.name].default
+            if default_val != inspect.Parameter.empty and default_val is not None:
+                try:
+                    _ = param_type.create_literal(default_val)
+                except TypeError as e:
+                    raise excs.Error(f'Default value for parameter {param.name}: {str(e)}')
+
+            parameters.append(Parameter(param.name, param_type, param.kind, is_batched))
+
+        return_is_batched = False
+        if return_type is None:
+            return_type, return_is_batched = cls._infer_type(sig.return_annotation)
+            if return_type is None:
+                raise excs.Error('Cannot infer pixeltable return type')
+        else:
+            _, return_is_batched = cls._infer_type(sig.return_annotation)
+
+        return Signature(return_type, parameters, return_is_batched)