clarifai 11.1.5rc7__py3-none-any.whl → 11.1.5rc8__py3-none-any.whl

clarifai/runners/utils/method_signatures.py

@@ -1,9 +1,9 @@
+import ast
 import inspect
 import json
-import re
-import types
-from collections import OrderedDict, namedtuple
-from typing import List, get_args, get_origin
+import textwrap
+from collections import namedtuple
+from typing import List, Tuple, get_args, get_origin
 
 import numpy as np
 import PIL.Image
@@ -12,12 +12,12 @@ from clarifai_grpc.grpc.api import resources_pb2
 from google.protobuf.message import Message as MessageProto
 
 from clarifai.runners.utils import data_types
-from clarifai.runners.utils.serializers import (AtomicFieldSerializer, ImageSerializer,
-                                                ListSerializer, MessageSerializer,
-                                                NDArraySerializer, NullValueSerializer, Serializer)
+from clarifai.runners.utils.serializers import (AtomicFieldSerializer, ListSerializer,
+                                                MessageSerializer, NamedFieldsSerializer,
+                                                NDArraySerializer, Serializer, TupleSerializer)
 
 
-def build_function_signature(func, method_type: str):
+def build_function_signature(func):
   '''
   Build a signature for the given function.
   '''
@@ -30,110 +30,144 @@ def build_function_signature(func, method_type: str):
 
   return_annotation = sig.return_annotation
   if return_annotation == inspect.Parameter.empty:
-    raise ValueError('Function must have a return annotation')
-  # check for multiple return values and convert to dict for named values
-  return_streaming = False
-  if get_origin(return_annotation) == data_types.Stream:
-    return_annotation = get_args(return_annotation)[0]
-    return_streaming = True
-  if get_origin(return_annotation) == tuple:
-    return_annotation = tuple(get_args(return_annotation))
-  if isinstance(return_annotation, tuple):
-    return_annotation = {'return.%s' % i: tp for i, tp in enumerate(return_annotation)}
-  if not isinstance(return_annotation, dict):
-    return_annotation = {'return': return_annotation}
-
-  input_vars = build_variables_signature(sig.parameters.values())
-  output_vars = build_variables_signature(
-      [
-          # XXX inspect.Parameter errors for the special return names, so use SimpleNamespace
-          types.SimpleNamespace(name=name, annotation=tp, default=inspect.Parameter.empty)
-          for name, tp in return_annotation.items()
-      ],
-      is_output=True)
-  if return_streaming:
-    for var in output_vars:
-      var.streaming = True
-
-  # check for streams
-  if method_type == 'predict':
-    for var in input_vars:
-      if var.streaming:
-        raise TypeError('Stream inputs are not supported for predict methods')
-    for var in output_vars:
-      if var.streaming:
-        raise TypeError('Stream outputs are not supported for predict methods')
-  elif method_type == 'generate':
-    for var in input_vars:
-      if var.streaming:
-        raise TypeError('Stream inputs are not supported for generate methods')
-    if not all(var.streaming for var in output_vars):
-      raise TypeError('Generate methods must return a stream')
-  elif method_type == 'stream':
-    input_stream_vars = [var for var in input_vars if var.streaming]
-    if len(input_stream_vars) == 0:
-      raise TypeError('Stream methods must include a Stream input')
-    if not all(var.streaming for var in output_vars):
-      raise TypeError('Stream methods must return a single Stream')
+    raise TypeError('Function must have a return annotation')
+
+  input_sigs = [
+      build_variable_signature(p.name, p.annotation, p.default) for p in sig.parameters.values()
+  ]
+  input_sigs, input_types, input_streaming = zip(*input_sigs)
+  output_sig, output_type, output_streaming = build_variable_signature(
+      'return', return_annotation, is_output=True)
+  # TODO: flatten out "return" layer if not needed
+
+  # check for streams and determine method type
+  if sum(input_streaming) > 1:
+    raise TypeError('streaming methods must have at most one streaming input')
+  input_streaming = any(input_streaming)
+  if not (input_streaming or output_streaming):
+    method_type = 'predict'
+  elif not input_streaming and output_streaming:
+    method_type = 'generate'
+  elif input_streaming and output_streaming:
+    method_type = 'stream'
   else:
-    raise TypeError('Invalid method type: %s' % method_type)
+    raise TypeError('stream methods with streaming inputs must have streaming outputs')
 
   #method_signature = resources_pb2.MethodSignature()   # TODO
-  method_signature = _NamedFields()  #for now
+  method_signature = _SignatureDict()  #for now
 
   method_signature.name = func.__name__
   #method_signature.method_type = getattr(resources_pb2.RunnerMethodType, method_type)
   assert method_type in ('predict', 'generate', 'stream')
   method_signature.method_type = method_type
   method_signature.docstring = func.__doc__
+  method_signature.annotations_json = json.dumps(_get_annotations_source(func))
 
   #method_signature.inputs.extend(input_vars)
   #method_signature.outputs.extend(output_vars)
-  method_signature.inputs = input_vars
-  method_signature.outputs = output_vars
+  method_signature.inputs = input_sigs
+  method_signature.outputs = output_sig
   return method_signature
 
 
-def build_variables_signature(parameters: List[inspect.Parameter], is_output=False):
+def _get_annotations_source(func):
+  """Extracts raw annotation strings from the function source."""
+  source = inspect.getsource(func)  # Get function source code
+  source = textwrap.dedent(source)  # Dedent source code
+  tree = ast.parse(source)  # Parse into AST
+  func_node = next(node for node in tree.body
+                   if isinstance(node, ast.FunctionDef))  # Get function node
+
+  annotations = {}
+  for arg in func_node.args.args:  # Process arguments
+    if arg.annotation:
+      annotations[arg.arg] = ast.unparse(arg.annotation)  # Get raw annotation string
+
+  if func_node.returns:  # Process return type
+    annotations["return"] = ast.unparse(func_node.returns)
+
+  return annotations
+
+
+def build_variable_signature(name, annotation, default=inspect.Parameter.empty, is_output=False):
   '''
-  Build a data proto signature for the given variable or return type annotation.
+  Build a data proto signature and get the normalized python type for the given annotation.
   '''
 
-  vars = []
-
   # check valid names (should already be constrained by python naming, but check anyway)
-  for param in parameters:
-    if not param.name.isidentifier() and not (is_output and
-                                              re.match(r'return(\.\d+)?', param.name)):
-      raise ValueError(f'Invalid variable name: {param.name}')
+  if not name.isidentifier():
+    raise ValueError(f'Invalid variable name: {name}')
 
   # get fields for each variable based on type
-  for param in parameters:
-    param_types, streaming = _normalize_types(param, is_output=is_output)
-
-    for name, tp in param_types.items():
-      #var = resources_pb2.MethodVariable()   # TODO
-      var = _NamedFields()
-      var.name = name
-      var.data_type = _DATA_TYPES[tp].data_type
-      var.data_field = _DATA_TYPES[tp].data_field
-      var.streaming = streaming
-      if not is_output:
-        var.required = (param.default is inspect.Parameter.empty)
-        if not var.required:
-          var.default = param.default
-      vars.append(var)
-
-  # check if any fields are used more than once, and if so, use parts
-  # also if more than one field uses parts lists, also use parts, since the lists can be different lengths
-  # NOTE this is a little fancy, another way would just be to check if there is more than one arg
-  fields_unique = (len(set(var.data_field for var in vars)) == len(vars))
-  num_parts_lists = sum(int(var.data_field.startswith('parts[]')) for var in vars)
-  if not fields_unique or num_parts_lists > 1:
-    for var in vars:
-      var.data_field = 'parts[%s].%s' % (var.name, var.data_field)
-
-  return vars
+  tp, streaming = _normalize_type(annotation)
+
+  #var = resources_pb2.VariableSignature()   # TODO
+  sig = _VariableSignature()  #for now
+  sig.name = name
+
+  _fill_signature_type(sig, tp)
+
+  sig.streaming = streaming
+
+  if not is_output:
+    sig.required = (default is inspect.Parameter.empty)
+    if not sig.required:
+      sig.default = default
+
+  return sig, type, streaming
+
+
+def _fill_signature_type(sig, tp):
+  try:
+    if tp in _DATA_TYPES:
+      sig.data_type = _DATA_TYPES[tp].data_type
+      return
+  except TypeError:
+    pass  # not hashable type
+
+  if isinstance(tp, data_types.NamedFields):
+    sig.data_type = DataType.NAMED_FIELDS
+    for name, inner_type in tp.items():
+      # inner_sig = sig.type_args.add()
+      sig.type_args.append(inner_sig := _VariableSignature())
+      inner_sig.name = name
+      _fill_signature_type(inner_sig, inner_type)
+    return
+
+  if get_origin(tp) == tuple:
+    sig.data_type = DataType.TUPLE
+    for inner_type in get_args(tp):
+      #inner_sig = sig.type_args.add()
+      sig.type_args.append(inner_sig := _VariableSignature())
+      _fill_signature_type(inner_sig, inner_type)
+    return
+
+  if get_origin(tp) == list:
+    sig.data_type = DataType.LIST
+    inner_type = get_args(tp)[0]
+    #inner_sig = sig.type_args.add()
+    sig.type_args.append(inner_sig := _VariableSignature())
+    _fill_signature_type(inner_sig, inner_type)
+    return
+
+  raise TypeError(f'Unsupported type: {tp}')
+
+
+def serializer_from_signature(signature):
+  '''
+    Get the serializer for the given signature.
+    '''
+  if signature.data_type in _SERIALIZERS_BY_TYPE_ENUM:
+    return _SERIALIZERS_BY_TYPE_ENUM[signature.data_type]
+  if signature.data_type == DataType.LIST:
+    return ListSerializer(serializer_from_signature(signature.type_args[0]))
+  if signature.data_type == DataType.TUPLE:
+    return TupleSerializer([serializer_from_signature(sig) for sig in signature.type_args])
+  if signature.data_type == DataType.NAMED_FIELDS:
+    return NamedFieldsSerializer(
+        {sig.name: serializer_from_signature(sig)
+         for sig in signature.type_args})
+  raise ValueError(f'Unsupported type: {signature.data_type}')
 
 
 def signatures_to_json(signatures):
@@ -143,7 +177,8 @@ def signatures_to_json(signatures):
 
 
 def signatures_from_json(json_str):
-  return json.loads(json_str, object_pairs_hook=_NamedFields)
+  d = json.loads(json_str, object_pairs_hook=_SignatureDict)
+  return d
 
 
 def signatures_to_yaml(signatures):
@@ -163,8 +198,6 @@ def serialize(kwargs, signatures, proto=None, is_output=False):
   '''
   if proto is None:
     proto = resources_pb2.Data()
-  if not is_output:  # TODO: use this consistently for return keys also
-    kwargs = flatten_nested_keys(kwargs, signatures, is_output)
   unknown = set(kwargs.keys()) - set(sig.name for sig in signatures)
   if unknown:
     if unknown == {'return'} and len(signatures) > 1:
@@ -177,11 +210,12 @@ def serialize(kwargs, signatures, proto=None, is_output=False):
         raise TypeError(f'Missing required argument: {sig.name}')
       continue  # skip missing fields, they can be set to default on the server
     data = kwargs[sig.name]
-    force_named_part = (_is_empty_proto_data(data) and not is_output and not sig.required)
-    data_proto, field = _get_data_part(
-        proto, sig, is_output=is_output, serializing=True, force_named_part=force_named_part)
-    serializer = get_serializer(sig.data_type)
-    serializer.serialize(data_proto, field, data)
+    serializer = serializer_from_signature(sig)
+    # TODO determine if any (esp the first) var can go in the proto without parts
+    # and whether to put this in the signature or dynamically determine it
+    part = proto.parts.add()
+    part.id = sig.name
+    serializer.serialize(part.data, data)
   return proto
 
 
@@ -189,82 +223,31 @@ def deserialize(proto, signatures, is_output=False):
   '''
   Deserialize the given proto into kwargs using the given signatures.
   '''
+  if isinstance(signatures, dict):
+    signatures = [signatures]  # TODO update return key level and make consistnet
   kwargs = {}
+  parts_by_name = {part.id: part for part in proto.parts}
   for sig in signatures:
-    data_proto, field = _get_data_part(proto, sig, is_output=is_output, serializing=False)
-    if data_proto is None:
-      # not set in proto, check if required or skip if optional arg
-      if not is_output and sig.required:
+    serializer = serializer_from_signature(sig)
+    part = parts_by_name.get(sig.name)
+    if part is None:
+      if sig.required or is_output:  # TODO allow optional outputs?
         raise ValueError(f'Missing required field: {sig.name}')
       continue
-    serializer = get_serializer(sig.data_type)
-    data = serializer.deserialize(data_proto, field)
-    kwargs[sig.name] = data
-  if is_output:
-    if len(kwargs) == 1 and 'return' in kwargs:  # case for single return value
-      return kwargs['return']
-    if kwargs and 'return.0' in kwargs:  # case for tuple return values
-      return tuple(kwargs[f'return.{i}'] for i in range(len(kwargs)))
-    return data_types.Output(kwargs)
-  kwargs = unflatten_nested_keys(kwargs, signatures, is_output)
+    kwargs[sig.name] = serializer.deserialize(part.data)
+  if len(kwargs) == 1 and 'return' in kwargs:
+    return kwargs['return']
   return kwargs
 
 
-def get_serializer(data_type: str) -> Serializer:
-  if data_type in _SERIALIZERS_BY_TYPE_STRING:
-    return _SERIALIZERS_BY_TYPE_STRING[data_type]
-  if data_type.startswith('List['):
-    inner_type_string = data_type[len('List['):-1]
-    inner_serializer = get_serializer(inner_type_string)
-    return ListSerializer(inner_serializer)
-  raise ValueError(f'Unsupported type: "{data_type}"')
-
-
-def flatten_nested_keys(kwargs, signatures, is_output):
-  '''
-  Flatten nested keys into a single key with a dot, e.g. {'a': {'b': 1}} -> {'a.b': 1}
-  in the kwargs, using the given signatures to determine which keys are nested.
-  '''
-  nested_keys = [sig.name for sig in signatures if '.' in sig.name]
-  outer_keys = set(key.split('.')[0] for key in nested_keys)
-  for outer in outer_keys:
-    if outer not in kwargs:
-      continue
-    kwargs.update({outer + '.' + k: v for k, v in kwargs.pop(outer).items()})
-  return kwargs
-
-
-def unflatten_nested_keys(kwargs, signatures, is_output):
+def get_stream_from_signature(signatures):
   '''
-  Unflatten nested keys in kwargs into a dict, e.g. {'a.b': 1} -> {'a': {'b': 1}}
-  Uses the signatures to determine which keys are nested.
-  The dict subclass is Input or Output, depending on the is_output flag.
-  Preserves the order of args from the signatures.
+  Get the stream signature from the given signatures.
   '''
-  unflattened = OrderedDict()
   for sig in signatures:
-    if '.' not in sig.name:
-      if sig.name in kwargs:
-        unflattened[sig.name] = kwargs[sig.name]
-      continue
-    if sig.name not in kwargs:
-      continue
-    parts = sig.name.split('.')
-    assert len(parts) == 2, 'Only one level of nested keys is supported'
-    if parts[0] not in unflattened:
-      unflattened[parts[0]] = data_types.Output() if is_output else data_types.Input()
-    unflattened[parts[0]][parts[1]] = kwargs[sig.name]
-  return unflattened
-
-
-def get_stream_from_signature(signatures):
-  streaming_signatures = [var for var in signatures if var.streaming]
-  if not streaming_signatures:
-    return None, []
-  stream_argname = set([var.name.split('.', 1)[0] for var in streaming_signatures])
-  assert len(stream_argname) == 1, 'streaming methods must have exactly one streaming function arg'
-  stream_argname = stream_argname.pop()
-  return stream_argname, streaming_signatures
+    if sig.streaming:
+      return sig
+  return None
 
 
 def _is_empty_proto_data(data):
@@ -275,178 +258,178 @@ def _is_empty_proto_data(data):
   return not data
 
 
-def _get_data_part(proto, sig, is_output, serializing, force_named_part=False):
-  field = sig.data_field
-
-  # check if we need to force a named part, to distinguish between empty and unset values
-  if force_named_part and not field.startswith('parts['):
-    field = f'parts[{sig.name}].{field}'
-
-  # gets the named part from the proto, according to the field path
-  # note we only support one level of named parts
-  #parts = field.replace(' ', '').split('.')
-  # split on . but not if it is inside brackets, e.g. parts[outer.inner].field
-  parts = re.split(r'\.(?![^\[]*\])', field.replace(' ', ''))
-
-  if len(parts) not in (1, 2, 3):  # field, parts[name].field, parts[name].parts[].field
-    raise ValueError('Invalid field: %s' % field)
-
-  if len(parts) == 1:
-    # also need to check if there is an explicitly named part, e.g. for empty values
-    part = next((part for part in proto.parts if part.id == sig.name), None)
-    if part:
-      return part.data, field
-    if not serializing and not is_output and _is_empty_proto_data(getattr(proto, field)):
-      return None, field
-    return proto, field
-
-  # list
-  if parts[0] == 'parts[]':
-    if len(parts) != 2:
-      raise ValueError('Invalid field: %s' % field)
-    return proto, field  # return the data that contains the list itself
-
-  # named part
-  if not (m := re.match(r'parts\[([\w.]+)\]', parts[0])):
-    raise ValueError('Invalid field: %s' % field)
-  if not (name := m.group(1)):
-    raise ValueError('Invalid field: %s' % field)
-  assert len(parts) in (2, 3)  # parts[name].field, parts[name].parts[].field
-  part = next((part for part in proto.parts if part.id == name), None)
-  if part is None:
-    if not serializing:
-      raise ValueError('Missing part: %s' % name)
-    part = proto.parts.add()
-    part.id = name
-  return part.data, '.'.join(parts[1:])
-
-
-def _normalize_types(param, is_output=False):
+def _normalize_type(tp):
   '''
-  Normalize the types for the given parameter.  Returns a dict of names to types,
-  including named return values for outputs, and a flag indicating if streaming is used.
+  Normalize the types for the given parameter.
+  Returns the normalized type and whether the parameter is streaming.
   '''
-  tp = param.annotation
-
   # stream type indicates streaming, not part of the data itself
+  # it can only be used at the top-level of the var type
   streaming = (get_origin(tp) == data_types.Stream)
   if streaming:
     tp = get_args(tp)[0]
 
-  if is_output or streaming:  # named types can be used for outputs or streaming inputs
-    # output type used for named return values, each with their own data type
-    if isinstance(tp, (dict, data_types.Output, data_types.Input)):
-      return {param.name + '.' + name: _normalize_data_type(val)
-              for name, val in tp.items()}, streaming
-    if tp == data_types.Output:  # check for Output type without values
-      if not is_output:
-        raise TypeError('Output types can only be used for output values')
-      raise TypeError('Output types must be instantiated with inner type values for each key')
-    if tp == data_types.Input:  # check for Output type without values
-      if is_output:
-        raise TypeError('Input types can only be used for input values')
-      raise TypeError(
-          'Stream[Input(...)] types must be instantiated with inner type values for each key')
-
-  return {param.name: _normalize_data_type(tp)}, streaming
+  return _normalize_data_type(tp), streaming
 
 
 def _normalize_data_type(tp):
   # check if list, and if so, get inner type
-  is_list = (get_origin(tp) == list)
-  if is_list:
+  if get_origin(tp) == list:
     tp = get_args(tp)[0]
+    return List[_normalize_data_type(tp)]
+
+  if isinstance(tp, (tuple, list)):
+    return Tuple[tuple(_normalize_data_type(val) for val in tp)]
+
+  if isinstance(tp, (dict, data_types.NamedFields)):
+    return data_types.NamedFields(**{name: _normalize_data_type(val) for name, val in tp.items()})
 
-  # check if numpy array, and if so, use ndarray
+  # check if numpy array type, and if so, use ndarray
   if get_origin(tp) == np.ndarray:
-    tp = np.ndarray
+    return np.ndarray
 
   # check for PIL images (sometimes types use the module, sometimes the class)
   # set these to use the Image data handler
-  if tp in (PIL.Image, PIL.Image.Image):
-    tp = data_types.Image
-
-  # put back list
-  if is_list:
-    tp = List[tp]
-
-  # check if supported type
-  if tp not in _DATA_TYPES:
-    raise ValueError(f'Unsupported type: {tp}')
+  if tp in (data_types.Image, PIL.Image, PIL.Image.Image):
+    return data_types.Image
+
+  # check for jsonable types
+  # TODO should we include dict vs list in the data type somehow?
+  if tp == dict or (get_origin(tp) == dict and tp not in _DATA_TYPES and _is_jsonable(tp)):
+    return data_types.JSON
+  if tp == list or (get_origin(tp) == list and tp not in _DATA_TYPES and _is_jsonable(tp)):
+    return data_types.JSON
+
+  # check for known data types
+  try:
+    if tp in _DATA_TYPES:
+      return tp
+  except TypeError:
+    pass  # not hashable type
+
+  raise TypeError(f'Unsupported type: {tp}')
+
+
+def _is_jsonable(tp):
+  if tp in (dict, list, tuple, str, int, float, bool, type(None)):
+    return True
+  if get_origin(tp) == list:
+    return _is_jsonable(get_args(tp)[0])
+  if get_origin(tp) == dict:
+    return all(_is_jsonable(val) for val in get_args(tp))
+  return False
+
+
+# TODO --- tmp classes to stand-in for protos until they are defined and built into this package
+class _SignatureDict(dict):
+  __getattr__ = dict.__getitem__
+  __setattr__ = dict.__setitem__
 
-  return tp
 
+class _VariableSignature(_SignatureDict):
 
-class _NamedFields(dict):
-  __getattr__ = dict.__getitem__
-  __setattr__ = dict.__setitem__
+  def __init__(self):
+    super().__init__()
+    self.name = ''
+    self.type = ''
+    self.type_args = []
+    self.streaming = False
+    self.required = False
+    self.default = ''
+    self.description = ''
 
 
 # data_type: name of the data type
 # data_field: name of the field in the data proto
 # serializer: serializer for the data type
-_DataType = namedtuple('_DataType', ('data_type', 'data_field', 'serializer'))
+_DataType = namedtuple('_DataType', ('data_type', 'serializer'))
+
+
+# this will come from the proto module, but for now, define it here
+class DataType:
+  NOT_SET = 'NOT_SET'
+
+  STR = 'STR'
+  BYTES = 'BYTES'
+  INT = 'INT'
+  FLOAT = 'FLOAT'
+  BOOL = 'BOOL'
+  NDARRAY = 'NDARRAY'
+  JSON = 'JSON'
+
+  TEXT = 'TEXT'
+  IMAGE = 'IMAGE'
+  CONCEPT = 'CONCEPT'
+  REGION = 'REGION'
+  FRAME = 'FRAME'
+  AUDIO = 'AUDIO'
+  VIDEO = 'VIDEO'
+
+  NAMED_FIELDS = 'NAMED_FIELDS'
+  TUPLE = 'TUPLE'
+  LIST = 'LIST'
 
-# mapping of supported python types to data type names, fields, and serializers
+
+# simple, non-container types that correspond directly to a data field
 _DATA_TYPES = {
     str:
-        _DataType('str', 'string_value', AtomicFieldSerializer()),
+        _DataType(DataType.STR, AtomicFieldSerializer('string_value')),
     bytes:
-        _DataType('bytes', 'bytes_value', AtomicFieldSerializer()),
+        _DataType(DataType.BYTES, AtomicFieldSerializer('bytes_value')),
     int:
-        _DataType('int', 'int_value', AtomicFieldSerializer()),
+        _DataType(DataType.INT, AtomicFieldSerializer('int_value')),
     float:
-        _DataType('float', 'float_value', AtomicFieldSerializer()),
+        _DataType(DataType.FLOAT, AtomicFieldSerializer('float_value')),
     bool:
-        _DataType('bool', 'bool_value', AtomicFieldSerializer()),
-    None:
-        _DataType('None', '', NullValueSerializer()),
+        _DataType(DataType.BOOL, AtomicFieldSerializer('bool_value')),
     np.ndarray:
-        _DataType('ndarray', 'ndarray', NDArraySerializer()),
+        _DataType(DataType.NDARRAY, NDArraySerializer('ndarray')),
     data_types.Text:
-        _DataType('Text', 'text', MessageSerializer(data_types.Text)),
+        _DataType(DataType.TEXT, MessageSerializer('text', data_types.Text)),
     data_types.Image:
-        _DataType('Image', 'image', ImageSerializer()),
+        _DataType(DataType.IMAGE, MessageSerializer('image', data_types.Image)),
     data_types.Concept:
-        _DataType('Concept', 'concepts', MessageSerializer(data_types.Concept)),
+        _DataType(DataType.CONCEPT, MessageSerializer('concepts', data_types.Concept)),
     data_types.Region:
-        _DataType('Region', 'regions', MessageSerializer(data_types.Region)),
+        _DataType(DataType.REGION, MessageSerializer('regions', data_types.Region)),
     data_types.Frame:
-        _DataType('Frame', 'frames', MessageSerializer(data_types.Frame)),
+        _DataType(DataType.FRAME, MessageSerializer('frames', data_types.Frame)),
     data_types.Audio:
-        _DataType('Audio', 'audio', MessageSerializer(data_types.Audio)),
+        _DataType(DataType.AUDIO, MessageSerializer('audio', data_types.Audio)),
     data_types.Video:
-        _DataType('Video', 'video', MessageSerializer(data_types.Video)),
-
-    # lists handled specially, not as generic lists using parts
-    List[int]:
-        _DataType('ndarray', 'ndarray', NDArraySerializer()),
-    List[float]:
-        _DataType('ndarray', 'ndarray', NDArraySerializer()),
-    List[bool]:
-        _DataType('ndarray', 'ndarray', NDArraySerializer()),
+        _DataType(DataType.VIDEO, MessageSerializer('video', data_types.Video)),
 }
 
+_SERIALIZERS_BY_TYPE_ENUM = {dt.data_type: dt.serializer for dt in _DATA_TYPES.values()}
 
-# add generic lists using parts, for all supported types
-def _add_list_fields():
-  for tp in list(_DATA_TYPES.keys()):
-    if List[tp] in _DATA_TYPES:
-      # already added as special case
-      continue
-
-    # check if data field is repeated, and if so, use repeated field for list
-    field_name = _DATA_TYPES[tp].data_field
-    descriptor = resources_pb2.Data.DESCRIPTOR.fields_by_name.get(field_name)
-    repeated = descriptor and descriptor.label == descriptor.LABEL_REPEATED
-
-    # add to supported types
-    data_type = 'List[%s]' % _DATA_TYPES[tp].data_type
-    data_field = field_name if repeated else 'parts[].' + field_name
-    serializer = ListSerializer(_DATA_TYPES[tp].serializer)
-
-    _DATA_TYPES[List[tp]] = _DataType(data_type, data_field, serializer)
 
+class CompatibilitySerializer(Serializer):
+  '''
+  Serialization of basic value types, used for backwards compatibility
+  with older models that don't have type signatures.
+  '''
 
-_add_list_fields()
-_SERIALIZERS_BY_TYPE_STRING = {dt.data_type: dt.serializer for dt in _DATA_TYPES.values()}
+  def serialize(self, data_proto, value):
+    tp = _normalize_data_type(type(value))
+
+    try:
+      serializer = _DATA_TYPES[tp].serializer
+    except (KeyError, TypeError):
+      raise TypeError(f'serializer currently only supports basic types, got {tp}')
+
+    serializer.serialize(data_proto, value)
+
+  def deserialize(self, data_proto):
+    fields = [k.name for k, _ in data_proto.ListFields()]
+    if 'parts' in fields:
+      raise ValueError('serializer does not support parts')
+    serializers = [
+        serializer for serializer in _SERIALIZERS_BY_TYPE_ENUM.values()
+        if serializer.field_name in fields
+    ]
+    if not serializers:
+      raise ValueError('Returned data not recognized')
+    if len(serializers) != 1:
+      raise ValueError('Only single output supported for serializer')
+    serializer = serializers[0]
+    return serializer.deserialize(data_proto)