mmif-python 1.0.10__tar.gz → 1.0.12__tar.gz

{mmif-python-1.0.10 → mmif-python-1.0.12}/MANIFEST.in

@@ -2,4 +2,5 @@
 include ./VERSION
 include ./requirements.txt
 include ./requirements.cv
+include ./requirements.seq
 

{mmif-python-1.0.10/mmif_python.egg-info → mmif-python-1.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mmif-python
-Version: 1.0.10
+Version: 1.0.12
 Summary: Python implementation of MultiMedia Interchange Format specification. (https://mmif.clams.ai)
 Home-page: https://mmif.clams.ai
 Author: Brandeis Lab for Linguistics and Computation
@@ -10,6 +10,21 @@ Description: ## MultiMedia Interchange Format
         [MMIF](https://mmif.clams.ai) is a JSON(-LD)-based data format designed for transferring annotation data between computational analysis applications in [CLAMS project](https://clams.ai). 
         
         
+        ## mmif-python
+        `mmif-python` is a Python implementation of the MMIF data format. 
+        `mmif-python` provides various helper classes and functions to handle MMIF JSON in Python, 
+        including ; 
+        
+        1. de-/serialization of MMIF internal data structures to/from JSON
+        2. validation of MMIF JSON
+        3. handling of CLAMS vocabulary types
+        4. navigation of MMIF object via various "search" methods (e.g. `mmif.get_all_views_contain(vocab_type))`)
+        
+        ## For more ...
+        * [Version history and patch notes](https://github.com/clamsproject/mmif-python/blob/main/CHANGELOG.md)
+        * [MMIF Python API documentation](https://clamsproject.github.io/mmif-python)
+        * [MMIF JSON specification and schema](https://clamsproject.github.io/mmif)
+        
 Platform: UNKNOWN
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers 
@@ -17,5 +32,6 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Provides-Extra: seq
 Provides-Extra: cv
 Provides-Extra: dev

mmif-python-1.0.12/README.md

@@ -0,0 +1,18 @@
+## MultiMedia Interchange Format
+[MMIF](https://mmif.clams.ai) is a JSON(-LD)-based data format designed for transferring annotation data between computational analysis applications in [CLAMS project](https://clams.ai). 
+
+
+## mmif-python
+`mmif-python` is a Python implementation of the MMIF data format. 
+`mmif-python` provides various helper classes and functions to handle MMIF JSON in Python, 
+including ; 
+
+1. de-/serialization of MMIF internal data structures to/from JSON
+2. validation of MMIF JSON
+3. handling of CLAMS vocabulary types
+4. navigation of MMIF object via various "search" methods (e.g. `mmif.get_all_views_contain(vocab_type))`)
+
+## For more ...
+* [Version history and patch notes](https://github.com/clamsproject/mmif-python/blob/main/CHANGELOG.md)
+* [MMIF Python API documentation](https://clamsproject.github.io/mmif-python)
+* [MMIF JSON specification and schema](https://clamsproject.github.io/mmif)

mmif-python-1.0.12/VERSION

@@ -0,0 +1 @@
+1.0.12

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/res/clams.vocabulary.yaml

@@ -78,6 +78,11 @@ properties:
       <br><br>
       [Note] Annotations from a classifier app must have this property.
   classifications:
+    type: Map from String to Number
+    description: >-
+      Alias for the <code>classification</code> metadata. Here for historical
+      reasons.
+  classification:
     type: Map from String to Number
     description: >-
       A map from label values to their "score" numbers provided by a classifier.

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/res/mmif.json

@@ -95,6 +95,9 @@
         },
         "parameters": {
           "$ref": "#/definitions/strStrMap"
+        },
+        "appConfiguration": {
+          "type": "object"
         }
       },
       "oneOf": [

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/serialize/annotation.py

@@ -18,16 +18,16 @@ from typing import Union, Dict, List, Optional, Iterator, MutableMapping, TypeVa
 from urllib.parse import urlparse
 
 from mmif.vocabulary import ThingTypesBase, DocumentTypesBase
-from .model import MmifObject, JSON_PRMTV_TYPES
+from .model import MmifObject, PRMTV_TYPES
 from .. import DocumentTypes, AnnotationTypes
 import mmif_docloc_http
 
 __all__ = ['Annotation', 'AnnotationProperties', 'Document', 'DocumentProperties', 'Text']
 
 T = TypeVar('T')
-LIST_PRMTV = typing.List[JSON_PRMTV_TYPES]  # list of values (most cases for annotation props)
+LIST_PRMTV = typing.List[PRMTV_TYPES]  # list of values (most cases for annotation props)
 LIST_LIST_PRMTV = typing.List[LIST_PRMTV]   # list of list of values (e.g. for coordinates)
-DICT_PRMTV = typing.Dict[str, JSON_PRMTV_TYPES]  # dict of values (`text` prop of `TextDocument` and other complex props)
+DICT_PRMTV = typing.Dict[str, PRMTV_TYPES]  # dict of values (`text` prop of `TextDocument` and other complex props)
 DICT_LIST_PRMTV = typing.Dict[str, LIST_PRMTV]  # dict of list of values (even more complex props)
 
 
@@ -77,7 +77,7 @@ class Annotation(MmifObject):
         subtypes, and effectively deprecated `frameType` and `boxType`
         in `TimeFrame` and `BoundingBox` respectively.
         """
-        prop_aliases = AnnotationTypes.prop_aliases.get(self._type.shortname, {})
+        prop_aliases = AnnotationTypes._prop_aliases.get(self._type.shortname, {})
         for alias_reprep, alias_group in prop_aliases.items():
             if key_to_add in alias_group:
                 for alias in alias_group:
@@ -129,10 +129,10 @@ class Annotation(MmifObject):
         
     @staticmethod
     def check_prop_value_is_simple_enough(
-            value: Union[JSON_PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> bool:
+            value: Union[PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> bool:
         
         def json_primitives(x): 
-            return isinstance(x, typing.get_args(JSON_PRMTV_TYPES))
+            return isinstance(x, typing.get_args(PRMTV_TYPES))
         
         def json_primitives_list(x):
             return isinstance(x, list) and all(map(json_primitives, x))
@@ -146,7 +146,7 @@ class Annotation(MmifObject):
             or (isinstance(value, dict) and all(map(lambda x: isinstance(x[0], str) and (json_primitives(x[1]) or json_primitives_list(x[1])), value.items())))
 
     def add_property(self, name: str,
-                     value: Union[JSON_PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> None:
+                     value: Union[PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> None:
         """
         Adds a property to the annotation's properties.
         :param name: the name of the property
@@ -162,7 +162,7 @@ class Annotation(MmifObject):
         #                      f"(\"{name}\": \"{str(value)}\"")
         self._add_prop_aliases(name, value)
 
-    def get(self, prop_name: str) -> Union['AnnotationProperties', JSON_PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]:
+    def get(self, prop_name: str) -> Union['AnnotationProperties', PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]:
         """
         A special getter for Annotation properties. This is to allow for
         directly accessing properties without having to go through the
@@ -242,7 +242,7 @@ class Document(Annotation):
         super().__init__(doc_obj)
     
     def add_property(self, name: str,
-                     value: Union[JSON_PRMTV_TYPES, LIST_PRMTV]
+                     value: Union[PRMTV_TYPES, LIST_PRMTV]
                      ) -> None:
         """
         Adds a property to the document's properties.

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/serialize/mmif.py

@@ -6,6 +6,7 @@ See the specification docs and the JSON Schema file for more information.
 """
 
 import json
+import math
 import warnings
 from collections import defaultdict
 from datetime import datetime
@@ -123,12 +124,20 @@ class Mmif(MmifObject):
                 # then, do the same to associated Document objects. Note that, 
                 # in a view, it is guaranteed that all Annotation objects are not duplicates
                 if ann.at_type == AnnotationTypes.Annotation:
+                    doc_id = ann.get_property('document')
                     try:
-                        doc_id = ann.get_property('document')
                         for prop_key, prop_value in ann.properties.items():
                             self.get_document_by_id(doc_id)._props_ephemeral[prop_key] = prop_value
                     except KeyError:
-                        warnings.warn(f"Annotation {ann.id} has a document ID {doc_id} that does not exist in the MMIF object. Skipping.", RuntimeWarning)
+                        warnings.warn(f"Annotation {ann.id} (in view {view.id}) has a document ID {doc_id} that "
+                                      f"does not exist in the MMIF object. Skipping.", RuntimeWarning)
+                # lastly, add quick access to `start` and `end` values if the annotation is using `targets` property
+                if 'targets' in ann.properties:
+                    if 'start' in ann.properties or 'end' in ann.properties:
+                        raise ValueError(f"Annotation {ann.id} (in view {view.id}) has `targes` and `start`/`end/` "
+                                         f"properties at the same time. Annotation anchors are ambiguous.")
+                    ann._props_ephemeral['start'] = self._get_linear_anchor_point(ann, start=True)
+                    ann._props_ephemeral['end'] = self._get_linear_anchor_point(ann, start=False)
 
     def generate_capital_annotations(self):
         """
@@ -427,7 +436,8 @@ class Mmif(MmifObject):
                 next(annotations)
                 views.append(view)
             except StopIteration:
-                # search failed by the full doc_id string, now try trimming the view_id from the string and re-do the search
+                # means search failed by the full doc_id string, 
+                # now try trimming the view_id from the string and re-do the search
                 if Mmif.id_delimiter in doc_id:
                     vid, did = doc_id.split(Mmif.id_delimiter)
                     if view.id == vid:
@@ -479,7 +489,7 @@ class Mmif(MmifObject):
                     return view
         return None
     
-    def _get_linear_anchor_point(self, ann: Annotation, start: bool = True) -> Union[int, float]:
+    def _get_linear_anchor_point(self, ann: Annotation, targets_sorted=False, start: bool = True) -> Union[int, float]:
         # TODO (krim @ 2/5/24): Update the return type once timeunits are unified to `ms` as integers (https://github.com/clamsproject/mmif/issues/192)
         """
         Retrieves the anchor point of the annotation. Currently, this method only supports linear anchors, 
@@ -487,21 +497,28 @@ class Mmif(MmifObject):
         
         :param ann: An Annotation object that has a linear anchor point. Namely, some subtypes of `Region` vocabulary type.
         :param start: If True, returns the start anchor point. Otherwise, returns the end anchor point. N/A for `timePoint` anchors.
+        :param targets_sorted: If True, the method will assume that the targets are sorted in the order of the anchor points.
         :return: the anchor point of the annotation. 1d for linear regions (time, text)
         """
         props = ann.properties
         if 'timePoint' in props:
             return ann.get_property('timePoint')
         elif 'targets' in props:
+            
+            def get_target_ann(cur_ann, target_id):
+                if Mmif.id_delimiter not in target_id:
+                    target_id = Mmif.id_delimiter.join((cur_ann.parent, target_id))
+                return self.__getitem__(target_id)
+            
+            if not targets_sorted:
+                point = math.inf if start else -1
+                comp = min if start else max
+                for target_id in ann.get_property('targets'):
+                    target = get_target_ann(ann, target_id)
+                    point = comp(point, self._get_linear_anchor_point(target, start=start))
+                return point
             target_id = ann.get_property('targets')[0 if start else -1]
-            # TODO (krim @ 2/5/24): not sure if this is the correct way to pick the "first" (or "last") target,
-            # since the targets list is not guaranteed to be sorted.
-            # However, due the recursive nature of this method, it is likely impossible
-            # to get all `start` (or `end`) values of the targets recursively and then pick the min (or max) value.
-            if Mmif.id_delimiter in target_id:
-                target = self.__getitem__(target_id)
-            else:
-                target = self.__getitem__(Mmif.id_delimiter.join((ann.parent, target_id)))
+            target = get_target_ann(ann, target_id)
             return self._get_linear_anchor_point(target, start=start)
         elif (start and 'start' in props) or (not start and 'end' in props):
             return ann.get_property('start' if start else 'end')

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/serialize/model.py

@@ -20,14 +20,14 @@ from deepdiff import DeepDiff
 
 T = TypeVar('T')
 S = TypeVar('S')
-JSON_PRMTV_TYPES: Type = Union[str, int, float, bool, None]
+PRMTV_TYPES: Type = Union[str, int, float, bool, None]
 
 __all__ = [
     'MmifObject',
     'MmifObjectEncoder',
     'DataList',
     'DataDict',
-    'JSON_PRMTV_TYPES'
+    'PRMTV_TYPES'
 ]
 
 

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/serialize/view.py

@@ -8,6 +8,7 @@ data that was previously present in the MMIF file.
 from datetime import datetime
 from typing import Dict, Union, Optional, Generator, List, cast
 
+from mmif.serialize.model import PRMTV_TYPES
 from mmif.vocabulary import ThingTypesBase, ClamsTypesBase
 from .annotation import Annotation, Document
 from .model import MmifObject, DataList, DataDict
@@ -243,7 +244,8 @@ class ViewMetadata(MmifObject):
         self.timestamp: Optional[datetime] = None
         self.app: str = ''
         self.contains: ContainsDict = ContainsDict()
-        self.parameters: dict = {}
+        self.parameters: Dict[str, str] = {}
+        self.appConfiguration: Dict[str, Union[PRMTV_TYPES, List[PRMTV_TYPES]]] = {}
         self.error: Union[dict, ErrorDict] = {}
         self.warnings: List[str] = []
         self._required_attributes = ["app"]
@@ -282,16 +284,43 @@ class ViewMetadata(MmifObject):
             self.add_contain(new_contain, at_type)
             return new_contain
     
-    def add_contain(self, contain: 'Contain', at_type: Union[str, ThingTypesBase]):
+    def add_contain(self, contain: 'Contain', at_type: Union[str, ThingTypesBase]) -> None:
         self.contains[at_type] = contain
 
-    def add_parameters(self, **runtime_params):
-        self.parameters.update(dict(runtime_params))
+    def add_app_configuration(self, config_key: str, config_value: Union[PRMTV_TYPES, List[PRMTV_TYPES]]) -> None:
+        """
+        Add a configuration key-value pair to the app_configuration dictionary.
+        """
+        self.appConfiguration[config_key] = config_value
+
+    def get_app_configuration(self, config_key: str) -> Union[PRMTV_TYPES, List[PRMTV_TYPES]]:
+        """
+        Get a configuration value from the app_configuration dictionary.
+        """
+        try:
+            return self.appConfiguration[config_key]
+        except KeyError:
+            raise KeyError(f"app is not configured for \"{config_key}\" key in the view: {self.serialize()}")
 
-    def add_parameter(self, param_key, param_value):
+    def add_parameters(self, **runtime_params: str):
+        """
+        Add runtime parameters as a batch (dict) to the view metadata. Note that parameter values must be strings.
+        """
+        for k, v in runtime_params.items():
+            self.add_parameter(k, v)
+
+    def add_parameter(self, param_key: str, param_value: str):
+        """
+        Add a single runtime parameter to the view metadata. Note that parameter value must be a string.
+        """
+        assert isinstance(param_value, str), \
+                f"Parameter value must be a string, \"{param_value}\" ({type(param_value)}) is given for key \"{param_key}\"."
         self.parameters[param_key] = param_value
 
-    def get_parameter(self, param_key):
+    def get_parameter(self, param_key: str) -> str:
+        """
+        Get a runtime parameter from the view metadata.
+        """
         try:
             return self.parameters[param_key]
         except KeyError:
@@ -384,7 +413,11 @@ class ContainsDict(DataDict[ThingTypesBase, Contain]):
         return self._items.get(key, default)
     
     def __contains__(self, item: Union[str, ThingTypesBase]):
-        return item in self._items
+        if isinstance(item, str):
+            string_keys = [str(k) for k in self._items.keys()]
+            return item in string_keys
+        else:
+            return item in self._items
 
     def pop(self, key):
         self._items.pop(key)

mmif-python-1.0.12/mmif/utils/sequence_helper.py

@@ -0,0 +1,292 @@
+"""
+This module provides helpers for handling sequence labeling. Specifically, it provides
+
+* a generalized label re-mapper for "post-binning" of labels
+* conversion from a list of CLAMS annotations (with ``classification`` props) into a list of reals (scores by labels), can be combined with the label re-mapper mentioned above
+* :py:meth:`mmif.utils.sequence_helper.smooth_outlying_short_intervals`: a simple smoothing algorithm by trimming "short" outlier sequences
+
+However, it DOES NOT provide 
+
+* direct conversion between CLAMS annotations. For example, it does not directly handle stitching of ``TimePoint`` into ``TimeFrames``. 
+* support for multi-class scenario, such as handling of _competing_ subsequence or overlapping labels.
+
+Some functions can use optional external libraries (e.g., ``numpy``) for better performance. 
+Hence, if you see a warning about missing optional packages, you might want to install them by running ``pip install mmif-python[seq]``.
+"""
+
+import importlib
+import itertools
+import warnings
+from typing import List, Tuple, Dict, Union, Iterable, Callable
+
+import mmif
+from mmif import Annotation
+from mmif.serialize.model import PRMTV_TYPES
+
+for seq_dep in ['numpy']:
+    try:
+        importlib.__import__(seq_dep)
+    except ImportError as e:
+        warnings.warn(f"Optional package \"{e.name}\" is not found. "
+                      f"You might want to install Sequence Helper dependencies "
+                      f"by running `pip install mmif-python[seq]=={mmif.__version__}`")
+
+NEG_LABEL = '-'
+
+
+def smooth_outlying_short_intervals(scores: List[float],
+                                    min_spseq_size: int,
+                                    min_snseq_size: int,
+                                    min_score: float = 0.5,
+                                    ):
+    """
+    Given a list of scores, a score threshold, and smoothing parameters, 
+    identify the intervals of "positive" scores by "trimming" the short 
+    positive sequences ("spseq") and short negative sequences ("snseq"). To 
+    decide the positivity, first step is binarization of the scores by the 
+    ``min_score`` threshold. Given ``Sr`` as "raw" input real-number scores 
+    list, and ``min_score=0.5``, 
+    
+        .. code-block:: javascript
+        
+            Sr: [0.3, 0.6, 0.2, 0.8, 0.2, 0.9, 0.8, 0.5, 0.1, 0.5, 0.8, 0.3, 1.0, 0.7, 0.5, 0.5, 0.5, 0.8, 0.3, 0.6]
+
+
+    
+    the binarization is done by simply comparing each score to the 
+    threshold to get ``S`` list of binary scores
+    
+        .. code-block:: javascript
+        
+            1.0 :                                     |                      
+            0.9 :                |                    |                      
+            0.8 :          |     |  |           |     |              |       
+            0.7 :          |     |  |           |     |  |           |       
+            0.6 :    |     |     |  |           |     |  |           |     | 
+            0.5 :----+-----+-----+--+--+-----+--+-----+--+--+--+--+--+-----+-
+            0.4 :    |     |     |  |  |     |  |     |  |  |  |  |  |     | 
+            0.3 : |  |     |     |  |  |     |  |  |  |  |  |  |  |  |  |  | 
+            0.2 : |  |  |  |  |  |  |  |     |  |  |  |  |  |  |  |  |  |  | 
+            0.1 : |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  | 
+            0.0 +------------------------------------------------------------
+            raw :.3 .6 .2 .8 .2 .9 .8 .5 .1 .5 .8 .3 1. .7 .5 .5 .5 .8 .3 .6
+             S  : 0  1  0  1  0  1  1  0  0  0  1  0  1  1  0  1  1  1  0  1 
+            
+    Note that the size of a positive or negative sequence can be as small 
+    as 1.
+       
+    Then, here are examples of smoothing a list of binary scores into 
+    intervals, by trimming "very short" (under thresholds) sequences of 
+    positive or negative:
+    
+    .. note::
+       legends: 
+       
+       * ``t`` is unit index (e.g. time index)
+       * ``S`` is the list of binary scores (zeros and ones)
+       * ``I`` is the list of intervals after smoothing
+    
+    #. with params ``min_spseq_size==1``, ``min_snseq_size==4``
+        
+        .. code-block:: javascript
+        
+           t: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+           S: [0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1]
+           I: [0, 1--1--1--1--1--1--1--1--1--1--1--1, 0--0--0--0--0--0, 1]
+        
+        Explanation: ``min_snseq_size`` is used to smooth short sequences 
+        of negative predictions. In this, zeros from t[7:10] are smoothed 
+        into "one" I, while zeros from t[13:19] are kept as "zero" I. Note 
+        that the "short" snseqs at the either ends (t[0:1]) are never 
+        smoothed.
+        
+    #. with params ``min_spseq_size==4``, ``min_snseq_size==2``
+    
+        .. code-block:: javascript
+        
+           t: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+           S: [0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1]
+           I: [0, 1--1--1--1--1--1, 0--0--0--0--0--0--0--0--0--0--0--0--0]
+        
+        Explanation: ``min_spseq_size`` is used to smooth short sequences 
+        of positive predictions. In this example, the spseqs of ones from 
+        both t[10:13] and t[19:20] are smoothed. Note that the "short" 
+        spseqs at the either ends (t[19:20]) are always smoothed.
+                
+    #. with params ``min_spseq_size==4``, ``min_snseq_size==4``
+    
+        .. code-block:: javascript
+        
+           t: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+           S: [0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1]
+           I: [0, 1--1--1--1--1--1--1--1--1--1--1--1--0--0--0--0--0--0--0]
+        
+        Explanation: When two threshold parameters are working together,
+        the algorithm will prioritize the smoothing of the snseqs over the 
+        smoothing of the spseqs. Thus, in this example, the snseq t[7:10] 
+        gets first smoothed "up" before the spseq t[10:13] is smoothed 
+        "down", resulting in a long final I.
+    
+    #. with params ``min_spseq_size==4``, ``min_snseq_size==4``
+    
+        .. code-block:: javascript
+        
+           t: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+           S: [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1]
+           I: [1--1--1--1--1--1--1, 0--0--0--0, 1--1--1--1--1--1--1--1--1]
+        
+        Explanation: Since smoothing of snseqs is prioritized, short spseqs 
+        at the beginning or the end can be kept. 
+    
+    #. with params ``min_spseq_size==1``, ``min_snseq_size==1``
+    
+        .. code-block:: javascript
+        
+           t: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+           S: [0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1]
+           I: [0--0--0, 1--1--1--1, 0--0--0--0, 1--1--1--1, 0--0--0, 1--1]
+        
+        Explanation: When both width thresholds are set to 1, the algorithm
+        works essentially in the "stitching" only mode.
+                 
+    :param scores: **SORTED** list of scores to be smoothed. The score list
+                   is assumed to be "exhaust" the entire time or space of 
+                   the underlying document segment.
+                   (Sorted by the start, and then by the end of anchors)
+    :param min_score: minimum threshold to use to discard 
+                      low-scored units (strictly less than)
+    :param min_spseq_size: minimum size of a positive sequence not to be 
+                          smoothed (greater or equal to)
+    :param min_snseq_size: minimum size of a negative sequence not to be
+                          smoothed (greater or equal to)
+    :return: list of tuples of start(inclusive)/end(exclusive) indices
+             of the "positive" sequences. Negative sequences (regardless of 
+             their size) are not included in the output.
+    """
+
+    def trimmer(elems, min_width, target=False, keep_short_ends=False):
+        """
+        This will loop through a list of bools and convert short ``target`` 
+        subsequences to ¬target (or not-target). A "short" subsequence is 
+        defined by its size being strictly smaller than ``min_width``.
+        When ``always_trim_ends``, the first and the last subsequences are 
+        always converted to ¬target, regardless of their size.
+        """
+        try:
+            assert min_width > 0
+        except AssertionError:
+            raise ValueError(f"minimum width threshold must be a positive number, but got {min_width}")
+        group_gen = itertools.groupby(elems)
+        new_membership = None
+        last_membership = None
+        last_membersnum = 0
+        added = 0
+        for i, (positivity, group) in enumerate(group_gen):
+            membersnum = len(list(group))
+            for _ in range(last_membersnum):  # won't be executed for the first group (last_membersnum=0)
+                yield new_membership
+                added += 1
+            new_membership = not target
+            # keep target as target if 
+            # 1. the sequence is wide enough
+            # 2. the sequence is at the beginning of the list and "keep_ends"
+            if positivity == target and (membersnum >= min_width or (keep_short_ends and i == 0)):
+                new_membership = target
+            last_membersnum = membersnum
+            last_membership = positivity
+        # and finally 3. the sequence is at the final of the list and "keep_ends"
+        new_membership = not target
+        if last_membership == target and (last_membersnum >= min_width or keep_short_ends):
+            new_membership = target
+        for _ in range(last_membersnum):
+            yield new_membership
+            added += 1
+
+    # first pass to smooth short gaps first
+    pass1 = trimmer(
+        map(lambda x: x >= min_score, scores),
+        min_snseq_size, target=False, keep_short_ends=True)
+    pass2 = trimmer(
+        pass1, min_spseq_size, target=True, keep_short_ends=False)
+    return _sequence_to_intervals(pass2)
+
+
+def _sequence_to_intervals(seq: Iterable[bool]) -> List[Tuple[int, int]]:
+    pos_ints = []
+    cur = 0
+    for positivity, members in itertools.groupby(seq):
+        l = len(list(members))
+        if positivity:
+            pos_ints.append((cur, cur + l))
+        cur += l
+    return pos_ints
+
+
+def validate_labelset(annotations: Iterable[Annotation]) -> List[str]:
+    """
+    Simple check for a list of annotations to see if they have the same label set.
+    
+    :raise: AttributeError if an element in the input list doesn't have the ``labelset`` property
+    :raise: ValueError if different ``labelset`` values are found
+    :return: a list of the common ``labelset`` value (list of label names)
+    """
+    # first, grab the label set from the source annotations
+    peek, annotations = itertools.tee(annotations)
+    try:
+        src_labels = set(next(peek).get_property('labelset'))
+    except KeyError:
+        raise AttributeError("The annotation in the list doesn't have "
+                             "'labelset' property. Are they annotated by"
+                             "a classification task?")
+
+    # and validate that all annotations have the same label set
+    for a in annotations:
+        if set(a.get_property('labelset')) != src_labels:
+            raise ValueError("All annotations must have the same label set, "
+                             f"but found {a.get_property('labelset')}, "
+                             f"different from {src_labels}")
+    return list(src_labels)
+
+
+def build_label_remapper(src_labels: List[str], dst_labels: Dict[str, PRMTV_TYPES]) -> Dict[str, PRMTV_TYPES]:
+    """
+    Build a label remapper dictionary from source and destination labels.
+    
+    :param src_labels: a list of all labels on the source side
+    :param dst_labels: a dict from source labels to destination labels. 
+                       Source labels not in this dict will be remapped to a negative label (``-``).
+    :return: a dict that exhaustively maps source labels to destination labels
+    """
+    if len(dst_labels) == 0:
+        return dict(zip(src_labels, src_labels))
+    else:
+        return {**dst_labels, **dict((k, NEG_LABEL) for k in src_labels if k not in dst_labels)}
+
+
+def build_score_lists(classifications: List[Dict], label_remapper: Dict,
+                      score_remap_op: Callable[..., float] = max) \
+        -> Tuple[Dict[str, int], "numpy.ndarray"]:  # pytype: disable=name-error
+
+    """
+    Build lists of scores indexed by the label names. 
+
+    :param classifications: list of dictionaries of classification results, taken from input annotation objects
+    :param label_remapper: a dictionary that maps source label names to destination label names (formerly "postbin")
+    :param score_remap_op: a function to remap the scores from multiple source labels binned to a destination label
+                            common choices are ``max``, ``min``, or ``sum``
+    :return: 1. a dictionary that maps label names to their index in the score list
+             2. 2-d numpy array of scores, of which rows are indexed by label map dict (first return value)
+    """
+    import numpy
+    scores = {lbl: [] for lbl in label_remapper.values()}
+    for c_idx, classification in enumerate(classifications):
+        for src_label, src_score in classification.items():
+            dst_label = label_remapper[src_label]
+            if len(scores[dst_label]) == c_idx:  # means this is the first score for this label for this loop iter
+                scores[dst_label].append(src_score)
+            else:
+                scores[dst_label][-1] = score_remap_op((scores[dst_label][-1], src_score))
+    label_idx = {label: i for i, label in enumerate(scores.keys())}
+    score_lists = list(scores.values())
+
+    return label_idx, numpy.array(score_lists)

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/utils/timeunit_helper.py

@@ -74,11 +74,11 @@ def convert(t: Union[int, float, str], in_unit: str, out_unit: str, fps: float)
             return int(t * fps)
     # s>(ms or i)
     elif in_unit == 'second':
-        return t * 1000 if out_unit == 'millisecond' else _second_to_isoformat(t)
+        return int(t * 1000) if out_unit == 'millisecond' else _second_to_isoformat(t)
     # ms>(s or i)
     elif in_unit == 'millisecond':
         return t / 1000 if out_unit == 'second' else _millisecond_to_isoformat(t)
     # f>ms, f>s
     else:
-        return (t / fps) if out_unit == 'second' else (t / fps * 1000)  # pytype: disable=bad-return-type
+        return (t / fps) if out_unit == 'second' else (round(t / fps, 3) * 1000)  # pytype: disable=bad-return-type
 

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/utils/video_document_helper.py

@@ -81,15 +81,19 @@ def extract_frames_as_images(video_document: Document, framenums: List[int], as_
         from PIL import Image
     frames = []
     video = capture(video_document)
-    for framenum in framenums:
-        if framenum > video_document.get_property(FRAMECOUNT_DOCPROP_KEY):
-            raise ValueError(f'Frame number {framenum} is greater than the number of frames in the video.')
-        video.set(cv2.CAP_PROP_POS_FRAMES, framenum-1)
+    cur_f = 0
+    while True:
+        if not framenums or cur_f > video_document.get_property(FRAMECOUNT_DOCPROP_KEY):
+            break
         ret, frame = video.read()
-        if ret:
+        if cur_f == framenums[0]:
+            if not ret:
+                sec = convert(cur_f, 'f', 's', video_document.get_property(FPS_DOCPROP_KEY))
+                warnings.warn(f'Frame #{cur_f} ({sec}s) could not be read from the video {video_document.id}.')
+                continue
             frames.append(Image.fromarray(frame[:, :, ::-1]) if as_PIL else frame)
-        else:
-            break
+            framenums.pop(0)
+        cur_f += 1
     return frames
 
 
@@ -104,7 +108,7 @@ def get_mid_framenum(mmif: Mmif, time_frame: Annotation):
     timeunit = time_frame.get_property('timeUnit')
     video_document = mmif[time_frame.get_property('document')]
     fps = get_framerate(video_document)
-    return convert(mmif.get_start(time_frame) + mmif.get_end(time_frame), timeunit, 'frame', fps) // 2
+    return convert(time_frame.get_property('start') + time_frame.get_property('end'), timeunit, 'frame', fps) // 2
 
 
 def extract_mid_frame(mmif: Mmif, time_frame: Annotation, as_PIL: bool = False):

mmif-python-1.0.12/mmif/ver/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "1.0.12"
+__specver__ = "1.0.4"

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/vocabulary/__init__.py

@@ -6,4 +6,4 @@ from .base_types import DocumentTypesBase
 from .annotation_types import AnnotationTypes
 from .document_types import DocumentTypes
 
-typevers = {**ThingType.typevers, **AnnotationTypes.typevers, **DocumentTypes.typevers}
+_typevers = {**ThingType._typevers, **AnnotationTypes._typevers, **DocumentTypes._typevers}

mmif-python-1.0.12/mmif/vocabulary/annotation_types.py

@@ -0,0 +1,43 @@
+# Spec version 1.0.4
+# This file is auto-generated by setup.py
+
+from .base_types import AnnotationTypesBase
+
+
+class AnnotationTypes(AnnotationTypesBase):
+    """
+    This class contains the CLAMS annotation types 
+    defined in the spec version 1.0.4 as class variables. 
+    """
+    
+    # prpoerty aliases, first added in MMIF 1.0.2
+    # map from type short name to a dictionary of property aliases
+    # where nested dictionary maps from representative name to its aliases
+    _prop_aliases = {
+        'TimeFrame': {'label': {'frameType', 'label'},
+                      'classification': {'classifications', 'classification'}},
+        'BoundingBox': {'label': {'boxType', 'label'}, 
+                        'classification': {'classifications', 'classification'}},
+        'Annotation': {'classification': {'classifications', 'classification'}},
+        'Region': {'classification': {'classifications', 'classification'}},
+        'Interval': {'classification': {'classifications', 'classification'}},
+        'Span': {'classification': {'classifications', 'classification'}},
+        'Chapter': {'classification': {'classifications', 'classification'}},
+        'Polygon': {'classification': {'classifications', 'classification'}},
+        'VideoObject': {'classification': {'classifications', 'classification'}},
+        'Relation': {'classification': {'classifications', 'classification'}},
+    }
+
+    Annotation = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Annotation/v5')
+    Region = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Region/v4')
+    TimePoint = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/TimePoint/v4')
+    Interval = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Interval/v4')
+    Span = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Span/v4')
+    TimeFrame = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/TimeFrame/v5')
+    Chapter = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Chapter/v5')
+    Polygon = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Polygon/v4')
+    BoundingBox = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/BoundingBox/v4')
+    VideoObject = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/VideoObject/v4')
+    Relation = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Relation/v4')
+    Alignment = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Alignment/v1')
+    _typevers = {'Annotation': 'v5', 'Region': 'v4', 'TimePoint': 'v4', 'Interval': 'v4', 'Span': 'v4', 'TimeFrame': 'v5', 'Chapter': 'v5', 'Polygon': 'v4', 'BoundingBox': 'v4', 'VideoObject': 'v4', 'Relation': 'v4', 'Alignment': 'v1'}

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/vocabulary/base_types.py

@@ -196,7 +196,7 @@ class DocumentTypesBase(ClamsTypesBase):
 class ThingType(ThingTypesBase):
     """
     This class contains the topmost CLAMS thing type 
-    defined in the spec version 1.0.2 as a class variable. 
+    defined in the spec version 1.0.4 as a class variable. 
     """
     Thing = ClamsTypesBase('http://mmif.clams.ai/vocabulary/Thing/v1')
-    typevers = {'Thing': 'v1'}
+    _typevers = {'Thing': 'v1'}

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif/vocabulary/document_types.py

@@ -1,4 +1,4 @@
-# Spec version 1.0.2
+# Spec version 1.0.4
 # This file is auto-generated by setup.py
 
 from .base_types import DocumentTypesBase
@@ -7,11 +7,11 @@ from .base_types import DocumentTypesBase
 class DocumentTypes(DocumentTypesBase):
     """
     This class contains the CLAMS document types 
-    defined in the spec version 1.0.2 as class variables. 
+    defined in the spec version 1.0.4 as class variables. 
     """
     Document = DocumentTypesBase('http://mmif.clams.ai/vocabulary/Document/v1')
     VideoDocument = DocumentTypesBase('http://mmif.clams.ai/vocabulary/VideoDocument/v1')
     AudioDocument = DocumentTypesBase('http://mmif.clams.ai/vocabulary/AudioDocument/v1')
     ImageDocument = DocumentTypesBase('http://mmif.clams.ai/vocabulary/ImageDocument/v1')
     TextDocument = DocumentTypesBase('http://mmif.clams.ai/vocabulary/TextDocument/v1')
-    typevers = {'Document': 'v1', 'VideoDocument': 'v1', 'AudioDocument': 'v1', 'ImageDocument': 'v1', 'TextDocument': 'v1'}
+    _typevers = {'Document': 'v1', 'VideoDocument': 'v1', 'AudioDocument': 'v1', 'ImageDocument': 'v1', 'TextDocument': 'v1'}

{mmif-python-1.0.10 → mmif-python-1.0.12/mmif_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mmif-python
-Version: 1.0.10
+Version: 1.0.12
 Summary: Python implementation of MultiMedia Interchange Format specification. (https://mmif.clams.ai)
 Home-page: https://mmif.clams.ai
 Author: Brandeis Lab for Linguistics and Computation
@@ -10,6 +10,21 @@ Description: ## MultiMedia Interchange Format
         [MMIF](https://mmif.clams.ai) is a JSON(-LD)-based data format designed for transferring annotation data between computational analysis applications in [CLAMS project](https://clams.ai). 
         
         
+        ## mmif-python
+        `mmif-python` is a Python implementation of the MMIF data format. 
+        `mmif-python` provides various helper classes and functions to handle MMIF JSON in Python, 
+        including ; 
+        
+        1. de-/serialization of MMIF internal data structures to/from JSON
+        2. validation of MMIF JSON
+        3. handling of CLAMS vocabulary types
+        4. navigation of MMIF object via various "search" methods (e.g. `mmif.get_all_views_contain(vocab_type))`)
+        
+        ## For more ...
+        * [Version history and patch notes](https://github.com/clamsproject/mmif-python/blob/main/CHANGELOG.md)
+        * [MMIF Python API documentation](https://clamsproject.github.io/mmif-python)
+        * [MMIF JSON specification and schema](https://clamsproject.github.io/mmif)
+        
 Platform: UNKNOWN
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers 
@@ -17,5 +32,6 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Provides-Extra: seq
 Provides-Extra: cv
 Provides-Extra: dev

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif_python.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ README.md
 setup.py
 ./VERSION
 ./requirements.cv
+./requirements.seq
 ./requirements.txt
 mmif/__init__.py
 mmif/res/__init__.py
@@ -16,6 +17,7 @@ mmif/serialize/mmif.py
 mmif/serialize/model.py
 mmif/serialize/view.py
 mmif/utils/__init__.py
+mmif/utils/sequence_helper.py
 mmif/utils/timeunit_helper.py
 mmif/utils/video_document_helper.py
 mmif/ver/__init__.py

{mmif-python-1.0.10 → mmif-python-1.0.12}/mmif_python.egg-info/requires.txt

@@ -11,3 +11,6 @@ pytest
 pytest-pep8
 pytest-cov
 pytype
+
+[seq]
+numpy

mmif-python-1.0.12/requirements.seq

@@ -0,0 +1 @@
+numpy

{mmif-python-1.0.10 → mmif-python-1.0.12}/setup.py

@@ -69,7 +69,7 @@ def generate_vocab_enum(spec_version, clams_types_vers, mod_name) -> str:
     for type_name, type_ver in clams_types_vers:
         vocab_url = f'http://mmif.clams.ai/vocabulary/{type_name}/{type_ver}'
         file_out.write(f"    {type_name} = {base_class_name}('{vocab_url}')\n")
-    file_out.write(f"    typevers = {dict(clams_types_vers)}\n")
+    file_out.write(f"    _typevers = {dict(clams_types_vers)}\n")
 
     string_out = file_out.getvalue()
     file_out.close()
@@ -134,7 +134,7 @@ def generate_vocabulary(spec_version, clams_types_vers):
             f"from .{mod_name} import {class_name}"
             for mod_name, classes in types.items()
             for class_name in classes
-        ) + '\n\n' + "typevers = {**ThingType.typevers, **AnnotationTypes.typevers, **DocumentTypes.typevers}" + '\n'
+        ) + '\n\n' + "_typevers = {**ThingType._typevers, **AnnotationTypes._typevers, **DocumentTypes._typevers}" + '\n'
     )
     document_types = []
     annotation_types = []
@@ -276,6 +276,9 @@ with open('requirements.txt') as requirements:
 with open('requirements.cv') as requirements:
     cv_requires = requirements.readlines()
 
+with open('requirements.seq') as requirements:
+    seq_requires = requirements.readlines()
+
 setuptools.setup(
     name=name,
     version=version,
@@ -294,6 +297,7 @@ setuptools.setup(
     },
     install_requires=requires,
     extras_require={
+        'seq': seq_requires,
         'cv': cv_requires,
         'dev': [
             'pytest',

mmif-python-1.0.10/README.md

@@ -1,3 +0,0 @@
-## MultiMedia Interchange Format
-[MMIF](https://mmif.clams.ai) is a JSON(-LD)-based data format designed for transferring annotation data between computational analysis applications in [CLAMS project](https://clams.ai). 
-

mmif-python-1.0.10/VERSION

@@ -1 +0,0 @@
-1.0.10

mmif-python-1.0.10/mmif/ver/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "1.0.10"
-__specver__ = "1.0.2"

mmif-python-1.0.10/mmif/vocabulary/annotation_types.py

@@ -1,33 +0,0 @@
-# Spec version 1.0.2
-# This file is auto-generated by setup.py
-
-from .base_types import AnnotationTypesBase
-
-
-class AnnotationTypes(AnnotationTypesBase):
-    """
-    This class contains the CLAMS annotation types 
-    defined in the spec version 1.0.2 as class variables. 
-    """
-    
-    # prpoerty aliases, first added in MMIF 1.0.2
-    # map from type short name to a dictionary of property aliases
-    # where nested dictionary maps from representative name to its aliases
-    prop_aliases = {
-        'TimeFrame': {'label': {'frameType', 'label'}},
-        'BoundingBox': {'label': {'boxType', 'label'}},
-    }
-
-    Annotation = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Annotation/v3')
-    Region = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Region/v2')
-    TimePoint = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/TimePoint/v2')
-    Interval = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Interval/v2')
-    Span = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Span/v2')
-    TimeFrame = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/TimeFrame/v3')
-    Chapter = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Chapter/v3')
-    Polygon = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Polygon/v2')
-    BoundingBox = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/BoundingBox/v2')
-    VideoObject = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/VideoObject/v2')
-    Relation = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Relation/v2')
-    Alignment = AnnotationTypesBase('http://mmif.clams.ai/vocabulary/Alignment/v1')
-    typevers = {'Annotation': 'v3', 'Region': 'v2', 'TimePoint': 'v2', 'Interval': 'v2', 'Span': 'v2', 'TimeFrame': 'v3', 'Chapter': 'v3', 'Polygon': 'v2', 'BoundingBox': 'v2', 'VideoObject': 'v2', 'Relation': 'v2', 'Alignment': 'v1'}