mmif-python 1.0.11__tar.gz → 1.0.15__tar.gz

mmif_python-1.0.15/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.1
+Name: mmif-python
+Version: 1.0.15
+Summary: Python implementation of MultiMedia Interchange Format specification. (https://mmif.clams.ai)
+Home-page: https://mmif.clams.ai
+Author: Brandeis Lab for Linguistics and Computation
+Author-email: admin@clams.ai
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers 
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: deepdiff>5
+Requires-Dist: jsonschema
+Provides-Extra: seq
+Requires-Dist: numpy; extra == "seq"
+Provides-Extra: cv
+Requires-Dist: pillow; extra == "cv"
+Requires-Dist: opencv-python; extra == "cv"
+Requires-Dist: ffmpeg-python; extra == "cv"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-pep8; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytype; extra == "dev"
+
+## MultiMedia Interchange Format
+[MMIF](https://mmif.clams.ai) is a JSON(-LD)-based data format designed for transferring annotation data between computational analysis applications in [CLAMS project](https://clams.ai). 
+
+
+## mmif-python
+`mmif-python` is a Python implementation of the MMIF data format. 
+`mmif-python` provides various helper classes and functions to handle MMIF JSON in Python, 
+including ; 
+
+1. de-/serialization of MMIF internal data structures to/from JSON
+2. validation of MMIF JSON
+3. handling of CLAMS vocabulary types
+4. navigation of MMIF object via various "search" methods (e.g. `mmif.get_all_views_contain(vocab_type))`)
+
+## For more ...
+* [Version history and patch notes](https://github.com/clamsproject/mmif-python/blob/main/CHANGELOG.md)
+* [MMIF Python API documentation](https://clamsproject.github.io/mmif-python)
+* [MMIF JSON specification and schema](https://clamsproject.github.io/mmif)

mmif_python-1.0.15/VERSION

@@ -0,0 +1 @@
+1.0.15

{mmif-python-1.0.11 → mmif_python-1.0.15}/mmif/serialize/annotation.py

@@ -26,7 +26,7 @@ __all__ = ['Annotation', 'AnnotationProperties', 'Document', 'DocumentProperties
 
 T = TypeVar('T')
 LIST_PRMTV = typing.List[PRMTV_TYPES]  # list of values (most cases for annotation props)
-LIST_LIST_PRMTV = typing.List[LIST_PRMTV]   # list of list of values (e.g. for coordinates)
+LIST_LIST_PRMTV = typing.List[LIST_PRMTV]   # list of lists of values (e.g. for coordinates)
 DICT_PRMTV = typing.Dict[str, PRMTV_TYPES]  # dict of values (`text` prop of `TextDocument` and other complex props)
 DICT_LIST_PRMTV = typing.Dict[str, LIST_PRMTV]  # dict of list of values (even more complex props)
 
@@ -127,6 +127,20 @@ class Annotation(MmifObject):
     def id(self, aid: str) -> None:
         self.properties.id = aid
         
+    @property
+    def long_id(self) -> str:
+        if self.parent is not None and len(self.parent) > 0:
+            return f"{self.parent}{self.id_delimiter}{self.id}"
+        else:
+            return self.id
+    
+    @long_id.setter
+    def long_id(self, long_id: str) -> None:
+        if self.id_delimiter in long_id:
+            self.parent, self.id = long_id.split(self.id_delimiter)
+        else:
+            self.id = long_id
+    
     @staticmethod
     def check_prop_value_is_simple_enough(
             value: Union[PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> bool:
@@ -401,26 +415,31 @@ class AnnotationProperties(MmifObject, MutableMapping[str, T]):
                 
     def __iter__(self) -> Iterator[str]:
         """
-        ``__iter__`` on Mapping should basically work as ``keys()`` method of vanilla dict
-        however, when MMIF objects are serialized, all optional (not in ``_req_atts``),
-        empty props are ignored (note that emtpy but required props are serialized 
-        with the *emtpy* value). 
-        Hence, this ``__iter__`` method should also work in the same way and 
-        ignore empty optional props. 
+        ``__iter__`` on Mapping should basically work as ``keys()`` method 
+        of vanilla dict.
         """
         for key in itertools.chain(self._named_attributes(), self._unnamed_attributes):
-            if key in self._required_attributes:
-                yield key
-            else:
-                try:
-                    self.__getitem__(key)
-                    yield key
-                except KeyError:
-                    pass
+            yield key
+                
+    def __getitem__(self, key):
+        """
+        Parent MmifObject class has a __getitem__ method that checks if 
+        the value is empty when asked for an unnamed attribute. But for 
+        AnnotationProperties, any arbitrary property that's added 
+        explicitly by the user (developer) should not be ignored and 
+        returned even the value is empty. 
+        """
+        if key in self._named_attributes():
+            return self.__dict__[key]
+        else:
+            return self._unnamed_attributes[key]
 
     def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None) -> None:
         self.id: str = ''
+        # any individual at_type (subclassing this class) can have its own set of required attributes
         self._required_attributes = ["id"]
+        # allowing additional attributes for arbitrary annotation properties
+        self._unnamed_attributes = {}
         super().__init__(mmif_obj)
 
 

{mmif-python-1.0.11 → mmif_python-1.0.15}/mmif/serialize/mmif.py

@@ -10,13 +10,13 @@ import math
 import warnings
 from collections import defaultdict
 from datetime import datetime
-from typing import List, Union, Optional, Dict, ClassVar, cast
+from typing import List, Union, Optional, Dict, cast
 
 import jsonschema.validators
+
 import mmif
 from mmif import ThingTypesBase
 from mmif.vocabulary import AnnotationTypes, DocumentTypes
-
 from .annotation import Annotation, Document
 from .model import MmifObject, DataList
 from .view import View
@@ -24,6 +24,104 @@ from .view import View
 __all__ = ['Mmif']
 
 
+class MmifMetadata(MmifObject):
+    """
+    Basic MmifObject class to contain the top-level metadata of a MMIF file.
+
+    :param metadata_obj: the JSON data
+    """
+
+    def __init__(self, metadata_obj: Optional[Union[bytes, str, dict]] = None) -> None:
+        # TODO (krim @ 10/7/20): there could be a better name and a better way to give a value to this
+        self.mmif: str = f"http://mmif.clams.ai/{mmif.__specver__}"
+        self._required_attributes = ["mmif"]
+        super().__init__(metadata_obj)
+
+
+class DocumentsList(DataList[Document]):
+    """
+    DocumentsList object that implements :class:`mmif.serialize.model.DataList`
+    for :class:`mmif.serialize.document.Document`.
+    """
+    _items: Dict[str, Document]
+
+    def _deserialize(self, input_list: list) -> None:  # pytype: disable=signature-mismatch
+        """
+        Extends base ``_deserialize`` method to initialize ``items`` as a dict from
+        document IDs to :class:`mmif.serialize.document.Document` objects.
+
+        :param input_list: the JSON data that defines the list of documents
+        :return: None
+        """
+        self._items = {item['properties']['id']: Document(item) for item in input_list}
+
+    def append(self, value: Document, overwrite=False) -> None:
+        """
+        Appends a document to the list.
+
+        Fails if there is already a document with the same ID
+        in the list, unless ``overwrite`` is set to True.
+
+        :param value: the :class:`mmif.serialize.document.Document`
+                      object to add
+        :param overwrite: if set to True, will overwrite an
+                          existing document with the same ID
+        :raises KeyError: if ``overwrite`` is set to False and
+                          a document with the same ID exists
+                          in the list
+        :return: None
+        """
+        super()._append_with_key(value.id, value, overwrite)
+
+
+class ViewsList(DataList[View]):
+    """
+    ViewsList object that implements :class:`mmif.serialize.model.DataList`
+    for :class:`mmif.serialize.view.View`.
+    """
+    _items: Dict[str, View]
+
+    def __init__(self, mmif_obj: Optional[Union[bytes, str, list]] = None):
+        super().__init__(mmif_obj)
+
+    def _deserialize(self, input_list: list) -> None:  # pytype: disable=signature-mismatch
+        """
+        Extends base ``_deserialize`` method to initialize ``items`` as a dict from
+        view IDs to :class:`mmif.serialize.view.View` objects.
+
+        :param input_list: the JSON data that defines the list of views
+        :return: None
+        """
+        if input_list:
+            self._items = {item['id']: View(item) for item in input_list}
+
+    def append(self, value: View, overwrite=False) -> None:
+        """
+        Appends a view to the list.
+
+        Fails if there is already a view with the same ID
+        in the list, unless ``overwrite`` is set to True.
+
+        :param value: the :class:`mmif.serialize.view.View`
+                      object to add
+        :param overwrite: if set to True, will overwrite an
+                          existing view with the same ID
+        :raises KeyError: if ``overwrite`` is set to False and
+                          a view with the same ID exists
+                          in the list
+        :return: None
+        """
+        super()._append_with_key(value.id, value, overwrite)
+
+    def get_last(self) -> Optional[View]:
+        """
+        Returns the last view appended to the list.
+        """
+        for view in reversed(self._items.values()):
+            if 'error' not in view.metadata and 'warning' not in view.metadata:
+                return view
+
+
 class Mmif(MmifObject):
     """
     MmifObject that represents a full MMIF file.
@@ -32,9 +130,6 @@ class Mmif(MmifObject):
     :param validate: whether to validate the data against the MMIF JSON schema.
     """
 
-    view_prefix: ClassVar[str] = 'v_'
-    id_delimiter: ClassVar[str] = ':'
-
     def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None, *, validate: bool = True) -> None:
         self.metadata: MmifMetadata = MmifMetadata()
         self.documents: DocumentsList = DocumentsList()
@@ -362,8 +457,8 @@ class Mmif(MmifObject):
         :return: a reference to the corresponding document, if it exists
         :raises KeyError: if there is no corresponding document
         """
-        if Mmif.id_delimiter in doc_id:
-            vid, did = doc_id.split(Mmif.id_delimiter)
+        if self.id_delimiter in doc_id:
+            vid, did = doc_id.split(self.id_delimiter)
             view = self[vid]
             if isinstance(view, View):
                 return view.get_document_by_id(did) 
@@ -410,8 +505,8 @@ class Mmif(MmifObject):
                     aligned_types = set()
                     for ann_id in [alignment['target'], alignment['source']]:
                         ann_id = cast(str, ann_id)
-                        if Mmif.id_delimiter in ann_id:
-                            view_id, ann_id = ann_id.split(Mmif.id_delimiter)
+                        if self.id_delimiter in ann_id:
+                            view_id, ann_id = ann_id.split(self.id_delimiter)
                             aligned_type = cast(Annotation, self[view_id][ann_id]).at_type
                         else:
                             aligned_type = cast(Annotation, alignment_view[ann_id]).at_type
@@ -438,8 +533,8 @@ class Mmif(MmifObject):
             except StopIteration:
                 # means search failed by the full doc_id string, 
                 # now try trimming the view_id from the string and re-do the search
-                if Mmif.id_delimiter in doc_id:
-                    vid, did = doc_id.split(Mmif.id_delimiter)
+                if self.id_delimiter in doc_id:
+                    vid, did = doc_id.split(self.id_delimiter)
                     if view.id == vid:
                         annotations = view.get_annotations(document=did)
                         try:
@@ -450,6 +545,16 @@ class Mmif(MmifObject):
                             pass
         return views
 
+    def get_all_views_with_error(self) -> List[View]:
+        """
+        Returns the list of all views in the MMIF that have errors. 
+        
+        :return: the list of views that contain errors but no annotations
+        """
+        return [v for v in self.views if v.has_error()]
+    
+    get_views_with_error = get_all_views_with_error
+            
     def get_all_views_contain(self, at_types: Union[ThingTypesBase, str, List[Union[str, ThingTypesBase]]]) -> List[View]:
         """
         Returns the list of all views in the MMIF if given types
@@ -464,11 +569,27 @@ class Mmif(MmifObject):
         else:
             return [view for view in self.views if at_types in view.metadata.contains]
 
-    def get_views_contain(self, at_types: Union[ThingTypesBase, str, List[Union[str, ThingTypesBase]]]) -> List[View]:
+    get_views_contain = get_all_views_contain
+    
+    def get_view_with_error(self) -> Optional[View]:
         """
-        An alias to `get_all_views_contain` method.
+        Returns the last view appended that contains an error.
+        
+        :return: the view, or None if no error is found
         """
-        return self.get_all_views_contain(at_types)
+        for view in reversed(self.views):
+            if view.has_error():
+                return view
+        return None
+    
+    def get_last_error(self) -> Optional[str]:
+        """
+        Returns the last error message found in the views.
+        
+        :return: the error message in human-readable format, or None if no error is found
+        """
+        v = self.get_view_with_error()
+        return v.get_error() if v is not None else None
 
     def get_view_contains(self, at_types: Union[ThingTypesBase, str, List[Union[str, ThingTypesBase]]]) -> Optional[View]:
         """
@@ -506,8 +627,8 @@ class Mmif(MmifObject):
         elif 'targets' in props:
             
             def get_target_ann(cur_ann, target_id):
-                if Mmif.id_delimiter not in target_id:
-                    target_id = Mmif.id_delimiter.join((cur_ann.parent, target_id))
+                if self.id_delimiter not in target_id:
+                    target_id = self.id_delimiter.join((cur_ann.parent, target_id))
                 return self.__getitem__(target_id)
             
             if not targets_sorted:
@@ -537,131 +658,41 @@ class Mmif(MmifObject):
         """
         return self._get_linear_anchor_point(annotation, start=False)
 
-    # pytype: disable=bad-return-type
-    def __getitem__(self, item: str) -> Union[Document, View, Annotation]:
+    def __getitem__(self, item: str) \
+            -> Union[Document, View, Annotation, MmifMetadata, DocumentsList, ViewsList]:
         """
-        getitem implementation for Mmif. When nothing is found, this will raise an error
-        rather than returning a None (although pytype doesn't think so...)
+        getitem implementation for Mmif. This will try to find any object, given an identifier or an immediate 
+        attribute name. When nothing is found, this will raise an error rather than returning a None 
 
         :raises KeyError: if the item is not found or if the search results are ambiguous
-        :param item: the search string, a document ID, a view ID, or a view-scoped annotation ID
+        :param item: an attribute name or an object identifier (a document ID, a view ID, or an annotation ID). When 
+                     annotation ID is given as a "short" ID (without view ID prefix), the method will try to find a 
+                     match from the first view, and return immediately if found.
         :return: the object searched for
+        :raise KeyError: if the item is not found or multiple objects are found with the same ID
         """
         if item in self._named_attributes():
             return self.__dict__[item]
-        split_attempt = item.split(Mmif.id_delimiter)
+        split_attempt = item.split(self.id_delimiter)
 
-        document_result = self.documents.get(split_attempt[0])
-        view_result = self.views.get(split_attempt[0])
+        found = []
 
         if len(split_attempt) == 1:
-            anno_result = None
-        elif view_result:
-            anno_result = view_result[split_attempt[1]]
+            found.append(self.documents.get(split_attempt[0]))
+            found.append(self.views.get(split_attempt[0]))
+            for view in self.views:
+                found.append(view.annotations.get(split_attempt[0]))
+        elif len(split_attempt) == 2:
+            v = self.get_view_by_id(split_attempt[0])
+            if v is not None:
+                found.append(v.annotations.get(split_attempt[1]))
         else:
             raise KeyError("Tried to subscript into a view that doesn't exist")
+        found = [x for x in found if x is not None]
 
-        if view_result and document_result:
+        if len(found) > 1:
             raise KeyError("Ambiguous ID search result")
-        if not (view_result or document_result):
+        elif len(found) == 0:
             raise KeyError("ID not found: %s" % item)
-        return anno_result or view_result or document_result
-    # pytype: enable=bad-return-type
-
-
-class MmifMetadata(MmifObject):
-    """
-    Basic MmifObject class to contain the top-level metadata of a MMIF file.
-
-    :param metadata_obj: the JSON data
-    """
-
-    def __init__(self, metadata_obj: Optional[Union[bytes, str, dict]] = None) -> None:
-        # TODO (krim @ 10/7/20): there could be a better name and a better way to give a value to this
-        self.mmif: str = f"http://mmif.clams.ai/{mmif.__specver__}"
-        self._required_attributes = ["mmif"]
-        super().__init__(metadata_obj)
-
-
-class DocumentsList(DataList[Document]):
-    """
-    DocumentsList object that implements :class:`mmif.serialize.model.DataList`
-    for :class:`mmif.serialize.document.Document`.
-    """
-    _items: Dict[str, Document]
-
-    def _deserialize(self, input_list: list) -> None:  # pytype: disable=signature-mismatch
-        """
-        Extends base ``_deserialize`` method to initialize ``items`` as a dict from
-        document IDs to :class:`mmif.serialize.document.Document` objects.
-
-        :param input_list: the JSON data that defines the list of documents
-        :return: None
-        """
-        self._items = {item['properties']['id']: Document(item) for item in input_list}
-
-    def append(self, value: Document, overwrite=False) -> None:
-        """
-        Appends a document to the list.
-
-        Fails if there is already a document with the same ID
-        in the list, unless ``overwrite`` is set to True.
-
-        :param value: the :class:`mmif.serialize.document.Document`
-                      object to add
-        :param overwrite: if set to True, will overwrite an
-                          existing document with the same ID
-        :raises KeyError: if ``overwrite`` is set to False and
-                          a document with the same ID exists
-                          in the list
-        :return: None
-        """
-        super()._append_with_key(value.id, value, overwrite)
-
-
-class ViewsList(DataList[View]):
-    """
-    ViewsList object that implements :class:`mmif.serialize.model.DataList`
-    for :class:`mmif.serialize.view.View`.
-    """
-    _items: Dict[str, View]
-    
-    def __init__(self, mmif_obj: Optional[Union[bytes, str, list]] = None):
-        super().__init__(mmif_obj)
-
-    def _deserialize(self, input_list: list) -> None:  # pytype: disable=signature-mismatch
-        """
-        Extends base ``_deserialize`` method to initialize ``items`` as a dict from
-        view IDs to :class:`mmif.serialize.view.View` objects.
-
-        :param input_list: the JSON data that defines the list of views
-        :return: None
-        """
-        if input_list:
-            self._items = {item['id']: View(item) for item in input_list}
-
-    def append(self, value: View, overwrite=False) -> None:
-        """
-        Appends a view to the list.
-
-        Fails if there is already a view with the same ID
-        in the list, unless ``overwrite`` is set to True.
-
-        :param value: the :class:`mmif.serialize.view.View`
-                      object to add
-        :param overwrite: if set to True, will overwrite an
-                          existing view with the same ID
-        :raises KeyError: if ``overwrite`` is set to False and
-                          a view with the same ID exists
-                          in the list
-        :return: None
-        """
-        super()._append_with_key(value.id, value, overwrite)
-
-    def get_last(self) -> Optional[View]:
-        """
-        Returns the last view appended to the list.
-        """
-        for view in reversed(self._items.values()):
-            if 'error' not in view.metadata and 'warning' not in view.metadata:
-                return view
+        else:
+            return found[-1]

{mmif-python-1.0.11 → mmif_python-1.0.15}/mmif/serialize/model.py

@@ -14,7 +14,7 @@ for the different components of MMIF is added in the subclasses.
 
 import json
 from datetime import datetime
-from typing import Union, Any, Dict, Optional, TypeVar, Generic, Generator, Iterator, Type, Set
+from typing import Union, Any, Dict, Optional, TypeVar, Generic, Generator, Iterator, Type, Set, ClassVar
 
 from deepdiff import DeepDiff
 
@@ -44,41 +44,48 @@ class MmifObject(object):
 
     1. _unnamed_attributes:
        Only can be either None or an empty dictionary. If it's set to None,
-       it means the class won't take any ``Additional Attributes`` in the JSON
-       schema sense. If it's a dict, users can throw any k-v pairs to the
-       class, EXCEPT for the reserved two key names.
+       it means the class won't take any ``Additional Attributes`` in the 
+       JSON schema sense. If it's an empty dict, users can throw any k-v 
+       pairs to the class, as long as the key is not a "reserved" name, 
+       and those additional attributes will be stored in this dict while
+       in memory. 
     2. _attribute_classes:
        This is a dict from a key name to a specific python class to use for
        deserialize the value. Note that a key name in this dict does NOT
        have to be a *named* attribute, but is recommended to be one.
     3. _required_attributes:
-       This is a simple list of names of attributes that are required in the object.
-       When serialize, an object will skip its *empty* (e.g. zero-length, or None)
-       attributes unless they are in this list. Otherwise, the serialized JSON
-       string would have empty representations (e.g. ``""``, ``[]``).
+       This is a simple list of names of attributes that are required in 
+       the object. When serialize, an object will skip its *empty* (e.g. 
+       zero-length, or None) attributes unless they are in this list. 
+       Otherwise, the serialized JSON string would have empty 
+       representations (e.g. ``""``, ``[]``).
     4. _exclude_from_diff:
-       This is a simple list of names of attributes that should be excluded from
-       the diff calculation in ``__eq__``. 
+       This is a simple list of names of attributes that should be excluded 
+       from the diff calculation in ``__eq__``. 
 
     # TODO (krim @ 8/17/20): this dict is however, a duplicate with the type hints in the class definition.
-    Maybe there is a better way to utilize type hints (e.g. getting them as a programmatically), but for now
-    developers should be careful to add types to hints as well as to this dict.
+    Maybe there is a better way to utilize type hints (e.g. getting them 
+    as a programmatically), but for now developers should be careful to 
+    add types to hints as well as to this dict.
 
     Also note that those special attributes MUST be set in the __init__()
     before calling super method, otherwise deserialization will not work.
 
     And also, a subclass that has one or more *named* attributes, it must
-    set those attributes in the __init__() before calling super method. When
-    serializing a MmifObject, all *empty* attributes will be ignored, so for
-    optional named attributes, you must leave the values empty (len == 0), but
-    NOT None. Any None-valued named attributes will cause issues with current
-    implementation.
+    set those attributes in the __init__() before calling super method. 
+    When serializing a MmifObject, all *empty* attributes will be ignored, 
+    so for optional named attributes, you must leave the values empty 
+    (len == 0), but NOT None. Any None-valued named attributes will cause 
+    issues with current implementation.
 
     :param mmif_obj: JSON string or `dict` to initialize an object.
      If not given, an empty object will be initialized, sometimes with
      an ID value automatically generated, based on its parent object.
     """
     
+    view_prefix: ClassVar[str] = 'v_'
+    id_delimiter: ClassVar[str] = ':'
+
     # these are the reserved names that cannot be used as attribute names, and 
     # they won't be serialized
     reserved_names: Set[str] = {
@@ -269,8 +276,8 @@ class MmifObject(object):
         """
         Returns number of attributes that are not *empty*. 
         """
-        return sum([named in self and not self.is_empty(self[named]) for named in self._named_attributes()]) \
-               + (len(self._unnamed_attributes) if self._unnamed_attributes else 0)
+        return (sum([named in self and not self.is_empty(self[named]) for named in self._named_attributes()]) 
+                + (len(self._unnamed_attributes) if self._unnamed_attributes else 0))
 
     def __setitem__(self, key, value) -> None:
         if key in self.reserved_names: