mmif-python 1.0.1__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mmif-python-1.0.1/mmif_python.egg-info → mmif-python-1.0.2}/PKG-INFO +1 -1
- mmif-python-1.0.2/VERSION +1 -0
- mmif-python-1.0.2/mmif/__init__.py +36 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/serialize/annotation.py +147 -38
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/serialize/mmif.py +124 -8
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/serialize/model.py +11 -6
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/serialize/view.py +7 -5
- mmif-python-1.0.2/mmif/ver/__init__.py +2 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2/mmif_python.egg-info}/PKG-INFO +1 -1
- mmif-python-1.0.1/VERSION +0 -1
- mmif-python-1.0.1/mmif/__init__.py +0 -20
- mmif-python-1.0.1/mmif/ver/__init__.py +0 -2
- {mmif-python-1.0.1 → mmif-python-1.0.2}/LICENSE +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/MANIFEST.in +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/README.md +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/res/__init__.py +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/res/clams.vocabulary.yaml +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/res/do-not-edit.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/res/mmif.json +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/serialize/__init__.py +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/ver/do-not-edit.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/vocabulary/__init__.py +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/vocabulary/annotation_types.py +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/vocabulary/base_types.py +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/vocabulary/do-not-edit.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif/vocabulary/document_types.py +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif_python.egg-info/SOURCES.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif_python.egg-info/dependency_links.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif_python.egg-info/requires.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/mmif_python.egg-info/top_level.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/requirements.txt +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/setup.cfg +0 -0
- {mmif-python-1.0.1 → mmif-python-1.0.2}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mmif-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: Python implementation of MultiMedia Interchange Format specification. (https://mmif.clams.ai)
|
|
5
5
|
Home-page: https://mmif.clams.ai
|
|
6
6
|
Author: Brandeis Lab for Linguistics and Computation
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.0.2
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import pkgutil
|
|
3
|
+
import re
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from typing import Dict, Type, Callable, Set
|
|
6
|
+
|
|
7
|
+
import pkg_resources
|
|
8
|
+
|
|
9
|
+
# DO NOT CHANGE THIS ORDER, important to prevent circular imports
|
|
10
|
+
from mmif.ver import __version__
|
|
11
|
+
from mmif.ver import __specver__
|
|
12
|
+
from mmif.vocabulary import *
|
|
13
|
+
from mmif.serialize import *
|
|
14
|
+
|
|
15
|
+
_res_pkg = 'res'
|
|
16
|
+
_ver_pkg = 'ver'
|
|
17
|
+
_vocabulary_pkg = 'vocabulary'
|
|
18
|
+
_schema_res_name = 'mmif.json'
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_mmif_json_schema():
|
|
22
|
+
res = pkg_resources.resource_stream(f'{__name__}.{_res_pkg}', _schema_res_name)
|
|
23
|
+
res_str = res.read().decode('utf-8')
|
|
24
|
+
res.close()
|
|
25
|
+
return res_str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
patches: Dict[Type, Set[Callable]] = defaultdict(set)
|
|
29
|
+
for _, name, ispkg in pkgutil.iter_modules():
|
|
30
|
+
if ispkg and re.match(r'mmif[-_]utils[-_]', name):
|
|
31
|
+
mod = importlib.import_module(name)
|
|
32
|
+
for c, ms in mod.patches.items():
|
|
33
|
+
for m in ms:
|
|
34
|
+
if m in patches[c]:
|
|
35
|
+
raise ValueError(f'Patch for {c}::{m.__name__} already exists.')
|
|
36
|
+
patches[c].add(m)
|
|
@@ -6,21 +6,28 @@ In MMIF, annotations are created by apps in a pipeline as a part
|
|
|
6
6
|
of a view. For documentation on how views are represented, see
|
|
7
7
|
:mod:`mmif.serialize.view`.
|
|
8
8
|
"""
|
|
9
|
+
import importlib
|
|
9
10
|
import itertools
|
|
10
11
|
import pathlib
|
|
11
|
-
|
|
12
|
+
import pkgutil
|
|
13
|
+
import re
|
|
14
|
+
import typing
|
|
15
|
+
import warnings
|
|
16
|
+
from typing import Union, Dict, List, Optional, Iterator, MutableMapping, TypeVar
|
|
12
17
|
from urllib.parse import urlparse
|
|
13
18
|
|
|
14
19
|
from mmif.vocabulary import ThingTypesBase, DocumentTypesBase
|
|
15
|
-
from .model import MmifObject
|
|
20
|
+
from .model import MmifObject, JSON_PRMTV_TYPES
|
|
16
21
|
|
|
17
22
|
__all__ = ['Annotation', 'AnnotationProperties', 'Document', 'DocumentProperties', 'Text']
|
|
18
23
|
|
|
19
24
|
T = TypeVar('T')
|
|
20
25
|
|
|
21
|
-
from .. import DocumentTypes
|
|
26
|
+
from .. import DocumentTypes, AnnotationTypes
|
|
22
27
|
|
|
23
|
-
|
|
28
|
+
discovered_docloc_plugins = {
|
|
29
|
+
name[len('mmif_docloc_'):]: importlib.import_module(name) for _, name, _ in pkgutil.iter_modules() if re.match(r'mmif[-_]docloc[-_]', name)
|
|
30
|
+
}
|
|
24
31
|
|
|
25
32
|
|
|
26
33
|
class Annotation(MmifObject):
|
|
@@ -30,6 +37,9 @@ class Annotation(MmifObject):
|
|
|
30
37
|
|
|
31
38
|
def __init__(self, anno_obj: Optional[Union[bytes, str, dict]] = None) -> None:
|
|
32
39
|
self._type: ThingTypesBase = ThingTypesBase('')
|
|
40
|
+
# to store the parent view ID
|
|
41
|
+
self._parent_view_id = ''
|
|
42
|
+
self.reserved_names.add('_parent_view_id')
|
|
33
43
|
if not hasattr(self, 'properties'): # don't overwrite DocumentProperties on super() call
|
|
34
44
|
self.properties: AnnotationProperties = AnnotationProperties()
|
|
35
45
|
self._attribute_classes = {'properties': AnnotationProperties}
|
|
@@ -62,6 +72,16 @@ class Annotation(MmifObject):
|
|
|
62
72
|
else:
|
|
63
73
|
self._type = at_type
|
|
64
74
|
|
|
75
|
+
@property
|
|
76
|
+
def parent(self) -> str:
|
|
77
|
+
return self._parent_view_id
|
|
78
|
+
|
|
79
|
+
@parent.setter
|
|
80
|
+
def parent(self, parent_view_id: str) -> None:
|
|
81
|
+
# I want to make this to accept `View` object as an input too,
|
|
82
|
+
# but import `View` will break the code due to circular imports
|
|
83
|
+
self._parent_view_id = parent_view_id
|
|
84
|
+
|
|
65
85
|
@property
|
|
66
86
|
def id(self) -> str:
|
|
67
87
|
return self.properties.id
|
|
@@ -69,32 +89,52 @@ class Annotation(MmifObject):
|
|
|
69
89
|
@id.setter
|
|
70
90
|
def id(self, aid: str) -> None:
|
|
71
91
|
self.properties.id = aid
|
|
92
|
+
|
|
93
|
+
@staticmethod
|
|
94
|
+
def check_prop_value_is_simple_enough(
|
|
95
|
+
value: Union[JSON_PRMTV_TYPES, List[JSON_PRMTV_TYPES], List[List[JSON_PRMTV_TYPES]]]):
|
|
96
|
+
def json_primitives(x):
|
|
97
|
+
return isinstance(x, typing.get_args(JSON_PRMTV_TYPES))
|
|
98
|
+
return json_primitives(value) \
|
|
99
|
+
or (isinstance(value, list) and all(map(json_primitives, value))) \
|
|
100
|
+
or (all(map(lambda elem: isinstance(elem, list), value)) and map(json_primitives, [subelem for elem in value for subelem in elem]))
|
|
72
101
|
|
|
73
102
|
def add_property(self, name: str,
|
|
74
|
-
value: Union[
|
|
75
|
-
|
|
76
|
-
List[List[JSON_COMPATIBLE_PRIMITIVES]]
|
|
77
|
-
]) -> None:
|
|
103
|
+
value: Union[JSON_PRMTV_TYPES, List[JSON_PRMTV_TYPES], List[List[JSON_PRMTV_TYPES]]]
|
|
104
|
+
) -> None:
|
|
78
105
|
"""
|
|
79
106
|
Adds a property to the annotation's properties.
|
|
80
107
|
:param name: the name of the property
|
|
81
108
|
:param value: the property's desired value
|
|
82
109
|
:return: None
|
|
83
110
|
"""
|
|
84
|
-
|
|
85
|
-
if json_primitives(value) or (
|
|
86
|
-
isinstance(value,list)
|
|
87
|
-
and all(map(json_primitives, value)) or (
|
|
88
|
-
all(map(lambda elem: isinstance(elem, list), value))
|
|
89
|
-
and map(json_primitives, [subelem for elem in value for subelem in elem])
|
|
90
|
-
)
|
|
91
|
-
):
|
|
111
|
+
if self.check_prop_value_is_simple_enough(value):
|
|
92
112
|
self.properties[name] = value
|
|
93
113
|
else:
|
|
94
114
|
raise ValueError("Property values cannot be a complex object. It must be "
|
|
95
115
|
"either string, number, boolean, None, or a list of them."
|
|
96
116
|
f"(\"{name}\": \"{str(value)}\"")
|
|
97
117
|
|
|
118
|
+
def get(self, prop_name: str) -> Union['AnnotationProperties', JSON_PRMTV_TYPES, List[JSON_PRMTV_TYPES], List[List[JSON_PRMTV_TYPES]]]:
|
|
119
|
+
"""
|
|
120
|
+
A special getter for Annotation properties. This is to allow for
|
|
121
|
+
directly accessing properties without having to go through the
|
|
122
|
+
properties object.
|
|
123
|
+
"""
|
|
124
|
+
if prop_name in {'at_type', '@type'}:
|
|
125
|
+
return str(self._type)
|
|
126
|
+
elif prop_name == 'properties':
|
|
127
|
+
return self.properties
|
|
128
|
+
elif prop_name in self.properties:
|
|
129
|
+
return self.properties[prop_name]
|
|
130
|
+
else:
|
|
131
|
+
raise KeyError(f"Property {prop_name} does not exist in this annotation.")
|
|
132
|
+
|
|
133
|
+
get_property = get
|
|
134
|
+
|
|
135
|
+
def __getitem__(self, prop_name: str):
|
|
136
|
+
return self.get(prop_name)
|
|
137
|
+
|
|
98
138
|
def is_document(self):
|
|
99
139
|
return isinstance(self.at_type, DocumentTypesBase)
|
|
100
140
|
|
|
@@ -113,36 +153,81 @@ class Document(Annotation):
|
|
|
113
153
|
:param document_obj: the JSON data that defines the document
|
|
114
154
|
"""
|
|
115
155
|
def __init__(self, doc_obj: Optional[Union[bytes, str, dict]] = None) -> None:
|
|
116
|
-
#
|
|
117
|
-
|
|
118
|
-
self.
|
|
156
|
+
# see https://github.com/clamsproject/mmif-python/issues/226 for discussion
|
|
157
|
+
# around the use of these three dictionaries
|
|
158
|
+
self._props_original: DocumentProperties = DocumentProperties()
|
|
159
|
+
self._props_existing: AnnotationProperties = AnnotationProperties()
|
|
160
|
+
self._props_temporary: AnnotationProperties = AnnotationProperties()
|
|
161
|
+
self.reserved_names.update(('_props_original', '_props_existing', '_props_temporary'))
|
|
119
162
|
|
|
120
163
|
self._type: Union[ThingTypesBase, DocumentTypesBase] = ThingTypesBase('')
|
|
121
|
-
self.properties
|
|
164
|
+
self.properties = self._props_original
|
|
122
165
|
self.disallow_additional_properties()
|
|
123
166
|
self._attribute_classes = {'properties': DocumentProperties}
|
|
124
167
|
super().__init__(doc_obj)
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
def parent(self, parent_view_id: str) -> None:
|
|
132
|
-
# I want to make this to accept `View` object as an input too,
|
|
133
|
-
# but import `View` will break the code due to circular imports
|
|
134
|
-
self._parent_view_id = parent_view_id
|
|
168
|
+
|
|
169
|
+
def _add_property_from_annotation(self, annotation: Annotation):
|
|
170
|
+
if annotation.at_type != AnnotationTypes.Annotation:
|
|
171
|
+
raise ValueError("Only `Annotation` type can be added as a property to a `Document` object.")
|
|
172
|
+
for prop_name, prop_value in annotation.properties.items():
|
|
173
|
+
self._props_existing[prop_name] = prop_value
|
|
135
174
|
|
|
136
175
|
def add_property(self, name: str,
|
|
137
|
-
value: Union[
|
|
138
|
-
|
|
176
|
+
value: Union[JSON_PRMTV_TYPES, List[JSON_PRMTV_TYPES]]
|
|
177
|
+
) -> None:
|
|
178
|
+
"""
|
|
179
|
+
Adds a property to the document's properties.
|
|
180
|
+
|
|
181
|
+
Unlike the parent :class:`Annotation` class, added properties of a
|
|
182
|
+
``Document`` object can be lost during serialization unless it belongs
|
|
183
|
+
to somewhere in a ``Mmif`` object. This is because we want to keep
|
|
184
|
+
``Document`` object as "read-only" as possible. Thus, if you want to add
|
|
185
|
+
a property to a ``Document`` object,
|
|
186
|
+
|
|
187
|
+
* add the document to a ``Mmif`` object (either in the documents list or
|
|
188
|
+
in a view from the views list), or
|
|
189
|
+
* directly write to ``Document.properties`` instead of using this method
|
|
190
|
+
(which is not recommended).
|
|
191
|
+
|
|
192
|
+
With the former method, the SDK will record the added property as a
|
|
193
|
+
`Annotation` annotation object, separate from the original `Document`
|
|
194
|
+
object. See :meth:`.Mmif.generate_capital_annotations()` for more.
|
|
195
|
+
|
|
196
|
+
A few notes to keep in mind:
|
|
197
|
+
|
|
198
|
+
#. You can't overwrite an existing property of a ``Document`` object.
|
|
199
|
+
#. A MMIF can have multiple ``Annotation`` objects with the same
|
|
200
|
+
property name but different values. When this happens, the SDK will
|
|
201
|
+
only keep the latest value (in order of appearances in views list) of
|
|
202
|
+
the property, effectively overwriting the previous values.
|
|
203
|
+
"""
|
|
139
204
|
if name == "text":
|
|
140
205
|
self.properties.text = Text(value)
|
|
206
|
+
elif name == "mime":
|
|
207
|
+
self.properties.mime = str(value)
|
|
141
208
|
elif name == "location":
|
|
142
209
|
self.location = value
|
|
210
|
+
elif name not in self._props_original:
|
|
211
|
+
if self.check_prop_value_is_simple_enough(value):
|
|
212
|
+
self._props_temporary[name] = value
|
|
213
|
+
else:
|
|
214
|
+
super().add_property(name, value)
|
|
215
|
+
|
|
216
|
+
def get(self, prop_name):
|
|
217
|
+
"""
|
|
218
|
+
A special getter for Document properties. This is to allow for reading
|
|
219
|
+
the three properties in a specific order so that the latest value is
|
|
220
|
+
returned, in case there are multiple values for the same key.
|
|
221
|
+
"""
|
|
222
|
+
if prop_name in self._props_temporary:
|
|
223
|
+
return self._props_temporary[prop_name]
|
|
224
|
+
elif prop_name in self._props_existing:
|
|
225
|
+
return self._props_existing[prop_name]
|
|
143
226
|
else:
|
|
144
|
-
super().
|
|
227
|
+
return super().get(prop_name)
|
|
145
228
|
|
|
229
|
+
get_property = get
|
|
230
|
+
|
|
146
231
|
@property
|
|
147
232
|
def text_language(self) -> str:
|
|
148
233
|
if self.at_type == DocumentTypes.TextDocument:
|
|
@@ -210,11 +295,12 @@ class Document(Annotation):
|
|
|
210
295
|
|
|
211
296
|
def location_path(self) -> Optional[str]:
|
|
212
297
|
"""
|
|
213
|
-
Retrieves
|
|
214
|
-
|
|
298
|
+
Retrieves a path that's resolved to a pathname in the local file system.
|
|
299
|
+
To obtain the original value of the "path" part in the location string
|
|
300
|
+
(before resolving), use ``properties.location_path_literal`` method.
|
|
215
301
|
Returns None when no location is set.
|
|
216
302
|
"""
|
|
217
|
-
return self.properties.
|
|
303
|
+
return self.properties.location_path_resolved()
|
|
218
304
|
|
|
219
305
|
|
|
220
306
|
class AnnotationProperties(MmifObject, MutableMapping[str, T]):
|
|
@@ -241,7 +327,7 @@ class AnnotationProperties(MmifObject, MutableMapping[str, T]):
|
|
|
241
327
|
empty props are ignored (note that emtpy but required props are serialized
|
|
242
328
|
with the *emtpy* value).
|
|
243
329
|
Hence, this ``__iter__`` method should also work in the same way and
|
|
244
|
-
|
|
330
|
+
ignore empty optional props.
|
|
245
331
|
"""
|
|
246
332
|
for key in itertools.chain(self._named_attributes(), self._unnamed_attributes):
|
|
247
333
|
if key in self._required_attributes:
|
|
@@ -349,9 +435,32 @@ class DocumentProperties(AnnotationProperties):
|
|
|
349
435
|
return "".join((parsed_location.netloc, parsed_location.path))
|
|
350
436
|
|
|
351
437
|
def location_path(self) -> Optional[str]:
|
|
438
|
+
warnings.warn('location_path() is deprecated. Use location_path_resolved() instead.', DeprecationWarning)
|
|
439
|
+
return self.location_path_resolved()
|
|
440
|
+
|
|
441
|
+
def location_path_resolved(self) -> Optional[str]:
|
|
442
|
+
"""
|
|
443
|
+
Retrieves only path name of the document location (hostname is ignored),
|
|
444
|
+
and then try to resolve the path name in the local file system.
|
|
445
|
+
This method should be used when the document scheme is ``file`` or empty.
|
|
446
|
+
For other schemes, users should install ``mmif-locdoc-<scheme>`` plugin.
|
|
447
|
+
|
|
448
|
+
Returns None when no location is set.
|
|
449
|
+
Raise ValueError when no code found to resolve the given location scheme.
|
|
450
|
+
"""
|
|
451
|
+
if self.location is None:
|
|
452
|
+
return None
|
|
453
|
+
scheme = self.location_scheme()
|
|
454
|
+
if scheme in ('', 'file'):
|
|
455
|
+
return urlparse(self.location).path
|
|
456
|
+
elif scheme in discovered_docloc_plugins:
|
|
457
|
+
return discovered_docloc_plugins[scheme].resolve(self.location)
|
|
458
|
+
else:
|
|
459
|
+
raise ValueError(f'Cannot resolve location of scheme "{scheme}". Interested in developing mmif-locdoc-{scheme} plugin? See https://clams.ai/mmif-python/plugins')
|
|
460
|
+
|
|
461
|
+
def location_path_literal(self) -> Optional[str]:
|
|
352
462
|
"""
|
|
353
463
|
Retrieves only path name of the document location (hostname is ignored).
|
|
354
|
-
Useful to get a path of a local file.
|
|
355
464
|
Returns None when no location is set.
|
|
356
465
|
"""
|
|
357
466
|
if self.location is None:
|
|
@@ -6,6 +6,8 @@ See the specification docs and the JSON Schema file for more information.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import json
|
|
9
|
+
import warnings
|
|
10
|
+
from collections import defaultdict
|
|
9
11
|
from datetime import datetime
|
|
10
12
|
from typing import List, Union, Optional, Dict, ClassVar, cast
|
|
11
13
|
|
|
@@ -68,11 +70,113 @@ class Mmif(MmifObject):
|
|
|
68
70
|
json_str = json.loads(json_str)
|
|
69
71
|
jsonschema.validators.validate(json_str, schema)
|
|
70
72
|
|
|
71
|
-
def serialize(self, pretty: bool = False, sanitize: bool = False) -> str:
|
|
73
|
+
def serialize(self, pretty: bool = False, sanitize: bool = False, autogenerate_capital_annotations=True) -> str:
|
|
74
|
+
"""
|
|
75
|
+
Serializes the MMIF object to a JSON string.
|
|
76
|
+
|
|
77
|
+
:param sanitize: If True, performs some sanitization of before returning
|
|
78
|
+
the JSON string. See :meth:`sanitize` for details.
|
|
79
|
+
:param autogenerate_capital_annotations: If True, automatically convert
|
|
80
|
+
any "pending" temporary properties from `Document` objects to
|
|
81
|
+
`Annotation` objects. See :meth:`generate_capital_annotations` for
|
|
82
|
+
details.
|
|
83
|
+
:param pretty: If True, returns string representation with indentation.
|
|
84
|
+
:return: JSON string of the MMIF object.
|
|
85
|
+
"""
|
|
86
|
+
if autogenerate_capital_annotations:
|
|
87
|
+
self.generate_capital_annotations()
|
|
88
|
+
# sanitization should be done after `Annotation` annotations are generated
|
|
72
89
|
if sanitize:
|
|
73
90
|
self.sanitize()
|
|
74
91
|
return super().serialize(pretty)
|
|
75
92
|
|
|
93
|
+
def _deserialize(self, input_dict: dict) -> None:
|
|
94
|
+
"""
|
|
95
|
+
Deserializes the MMIF JSON string into a Mmif object.
|
|
96
|
+
This will read in existing ``Annotation`` typed annotations and
|
|
97
|
+
attach the document-level properties to the ``Document`` objects,
|
|
98
|
+
using a volatile property dict. This will allow apps to access the
|
|
99
|
+
document-level properties without having too much hassle to iterate
|
|
100
|
+
views and manually collect the properties.
|
|
101
|
+
"""
|
|
102
|
+
super()._deserialize(input_dict)
|
|
103
|
+
for view in self.views:
|
|
104
|
+
doc_id = None
|
|
105
|
+
if AnnotationTypes.Annotation in view.metadata.contains:
|
|
106
|
+
if 'document' in view.metadata.contains[AnnotationTypes.Annotation]:
|
|
107
|
+
doc_id = view.metadata.contains[AnnotationTypes.Annotation]['document']
|
|
108
|
+
# in a view, it is guaranteed that all Annotation objects are not duplicates
|
|
109
|
+
for ann in view.get_annotations(AnnotationTypes.Annotation):
|
|
110
|
+
if doc_id is None:
|
|
111
|
+
doc_id = ann.get_property('document')
|
|
112
|
+
try:
|
|
113
|
+
self.get_document_by_id(doc_id)._add_property_from_annotation(ann)
|
|
114
|
+
except KeyError:
|
|
115
|
+
warnings.warn(f"Annotation {ann.id} has a document ID {doc_id} that does not exist in the MMIF object. Skipping.", RuntimeWarning)
|
|
116
|
+
|
|
117
|
+
def generate_capital_annotations(self):
|
|
118
|
+
"""
|
|
119
|
+
Automatically convert any "pending" temporary properties from
|
|
120
|
+
`Document` objects to `Annotation` objects . The generated `Annotation`
|
|
121
|
+
objects are then added to the last `View` in the views lists.
|
|
122
|
+
|
|
123
|
+
See https://github.com/clamsproject/mmif-python/issues/226 for rationale
|
|
124
|
+
behind this behavior and discussion.
|
|
125
|
+
"""
|
|
126
|
+
# this view will be the default kitchen sink for all generated annotations
|
|
127
|
+
last_view = self.views.get_last()
|
|
128
|
+
# proceed only when there's at least one view
|
|
129
|
+
if last_view:
|
|
130
|
+
# to avoid duplicate property recording, this will be populated with
|
|
131
|
+
# existing Annotation objects from all existing views
|
|
132
|
+
existing_anns = defaultdict(lambda: defaultdict(dict))
|
|
133
|
+
|
|
134
|
+
# new properties to record in the current serialization call
|
|
135
|
+
anns_to_write = defaultdict(dict)
|
|
136
|
+
for view in self.views:
|
|
137
|
+
doc_id = None
|
|
138
|
+
if AnnotationTypes.Annotation in view.metadata.contains:
|
|
139
|
+
if 'document' in view.metadata.contains[AnnotationTypes.Annotation]:
|
|
140
|
+
doc_id = view.metadata.contains[AnnotationTypes.Annotation]['document']
|
|
141
|
+
for ann in view.get_annotations(AnnotationTypes.Annotation):
|
|
142
|
+
if doc_id is None:
|
|
143
|
+
doc_id = ann.get_property('document')
|
|
144
|
+
existing_anns[doc_id].update(ann.properties)
|
|
145
|
+
for doc in view.get_documents():
|
|
146
|
+
anns_to_write[doc.id].update(doc._props_temporary)
|
|
147
|
+
for doc in self.documents:
|
|
148
|
+
anns_to_write[doc.id].update(doc._props_temporary)
|
|
149
|
+
# additional iteration of views, to find a proper view to add the
|
|
150
|
+
# generated annotations. If none found, use the last view as the kitchen sink
|
|
151
|
+
last_view_for_docs = defaultdict(lambda: last_view)
|
|
152
|
+
doc_ids = set(anns_to_write.keys())
|
|
153
|
+
for doc_id in doc_ids:
|
|
154
|
+
for view in reversed(self.views):
|
|
155
|
+
# first try to find out if this view "contains" any annotation to the doc
|
|
156
|
+
# then, check for individual annotations
|
|
157
|
+
if [cont for cont in view.metadata.contains.values() if cont.get('document', None) == doc_id] \
|
|
158
|
+
or list(view.get_annotations(document=doc_id)):
|
|
159
|
+
last_view_for_docs[doc_id] = view
|
|
160
|
+
break
|
|
161
|
+
for doc_id, found_props in anns_to_write.items():
|
|
162
|
+
# ignore the "empty" id property from temporary dict
|
|
163
|
+
# `id` is "required" attribute for `AnnotationProperty` class
|
|
164
|
+
# thus will always be present in `props` dict as a key with emtpy value
|
|
165
|
+
# also ignore duplicate k-v pairs
|
|
166
|
+
props = {}
|
|
167
|
+
for k, v in found_props.items():
|
|
168
|
+
if k != 'id' and existing_anns[doc_id][k] != v:
|
|
169
|
+
props[k] = v
|
|
170
|
+
if props:
|
|
171
|
+
if len(anns_to_write) == 1:
|
|
172
|
+
# if there's only one document, we can record the doc_id in the contains metadata
|
|
173
|
+
last_view_for_docs[doc_id].metadata.new_contain(AnnotationTypes.Annotation, document=doc_id)
|
|
174
|
+
props.pop('document', None)
|
|
175
|
+
else:
|
|
176
|
+
# otherwise, doc_id needs to be recorded in the annotation property
|
|
177
|
+
props['document'] = doc_id
|
|
178
|
+
last_view_for_docs[doc_id].new_annotation(AnnotationTypes.Annotation, **props)
|
|
179
|
+
|
|
76
180
|
def sanitize(self):
|
|
77
181
|
"""
|
|
78
182
|
Sanitizes a Mmif object by running some safeguards.
|
|
@@ -195,9 +299,9 @@ class Mmif(MmifObject):
|
|
|
195
299
|
docs = []
|
|
196
300
|
for view in self.views:
|
|
197
301
|
for doc in view.get_documents():
|
|
198
|
-
if prop_key in doc and doc.
|
|
302
|
+
if prop_key in doc and doc.get(prop_key) == prop_value:
|
|
199
303
|
docs.append(doc)
|
|
200
|
-
docs.extend([document for document in self.documents if document
|
|
304
|
+
docs.extend([document for document in self.documents if document[prop_key] == prop_value])
|
|
201
305
|
return docs
|
|
202
306
|
|
|
203
307
|
def get_documents_locations(self, m_type: Union[DocumentTypes, str], path_only=False) -> List[Union[str, None]]:
|
|
@@ -231,7 +335,7 @@ class Mmif(MmifObject):
|
|
|
231
335
|
|
|
232
336
|
:param doc_id: the ID to search for
|
|
233
337
|
:return: a reference to the corresponding document, if it exists
|
|
234
|
-
:raises
|
|
338
|
+
:raises KeyError: if there is no corresponding document
|
|
235
339
|
"""
|
|
236
340
|
if Mmif.id_delimiter in doc_id:
|
|
237
341
|
vid, did = doc_id.split(Mmif.id_delimiter)
|
|
@@ -279,7 +383,7 @@ class Mmif(MmifObject):
|
|
|
279
383
|
else:
|
|
280
384
|
for alignment in alignment_view.get_annotations(AnnotationTypes.Alignment):
|
|
281
385
|
aligned_types = set()
|
|
282
|
-
for ann_id in [alignment
|
|
386
|
+
for ann_id in [alignment['target'], alignment['source']]:
|
|
283
387
|
ann_id = cast(str, ann_id)
|
|
284
388
|
if Mmif.id_delimiter in ann_id:
|
|
285
389
|
view_id, ann_id = ann_id.split(Mmif.id_delimiter)
|
|
@@ -294,10 +398,10 @@ class Mmif(MmifObject):
|
|
|
294
398
|
v_and_a[alignment_view.id] = alignments
|
|
295
399
|
return v_and_a
|
|
296
400
|
|
|
297
|
-
def get_views_for_document(self, doc_id: str):
|
|
401
|
+
def get_views_for_document(self, doc_id: str) -> List[View]:
|
|
298
402
|
"""
|
|
299
403
|
Returns the list of all views that have annotations anchored on a particular document.
|
|
300
|
-
Note that when the document is
|
|
404
|
+
Note that when the document is inside a view (generated during the pipeline's running),
|
|
301
405
|
doc_id must be prefixed with the view_id.
|
|
302
406
|
"""
|
|
303
407
|
views = []
|
|
@@ -447,6 +551,9 @@ class ViewsList(DataList[View]):
|
|
|
447
551
|
for :class:`mmif.serialize.view.View`.
|
|
448
552
|
"""
|
|
449
553
|
_items: Dict[str, View]
|
|
554
|
+
|
|
555
|
+
def __init__(self, mmif_obj: Optional[Union[bytes, str, list]] = None):
|
|
556
|
+
super().__init__(mmif_obj)
|
|
450
557
|
|
|
451
558
|
def _deserialize(self, input_list: list) -> None: # pytype: disable=signature-mismatch
|
|
452
559
|
"""
|
|
@@ -456,7 +563,8 @@ class ViewsList(DataList[View]):
|
|
|
456
563
|
:param input_list: the JSON data that defines the list of views
|
|
457
564
|
:return: None
|
|
458
565
|
"""
|
|
459
|
-
|
|
566
|
+
if input_list:
|
|
567
|
+
self._items = {item['id']: View(item) for item in input_list}
|
|
460
568
|
|
|
461
569
|
def append(self, value: View, overwrite=False) -> None:
|
|
462
570
|
"""
|
|
@@ -475,3 +583,11 @@ class ViewsList(DataList[View]):
|
|
|
475
583
|
:return: None
|
|
476
584
|
"""
|
|
477
585
|
super()._append_with_key(value.id, value, overwrite)
|
|
586
|
+
|
|
587
|
+
def get_last(self) -> Optional[View]:
|
|
588
|
+
"""
|
|
589
|
+
Returns the last view appended to the list.
|
|
590
|
+
"""
|
|
591
|
+
for view in reversed(self._items.values()):
|
|
592
|
+
if 'error' not in view.metadata and 'warning' not in view.metadata:
|
|
593
|
+
return view
|
|
@@ -11,21 +11,25 @@ core functionality for deserializing MMIF JSON data into live objects
|
|
|
11
11
|
and serializing live objects into MMIF JSON data. Specialized behavior
|
|
12
12
|
for the different components of MMIF is added in the subclasses.
|
|
13
13
|
"""
|
|
14
|
-
|
|
15
14
|
import json
|
|
16
15
|
from datetime import datetime
|
|
16
|
+
from types import MethodType
|
|
17
17
|
from typing import Union, Any, Dict, Optional, TypeVar, Generic, Generator, Iterator, Type, Set
|
|
18
18
|
|
|
19
19
|
from deepdiff import DeepDiff
|
|
20
20
|
|
|
21
|
+
import mmif
|
|
22
|
+
|
|
21
23
|
T = TypeVar('T')
|
|
22
24
|
S = TypeVar('S')
|
|
25
|
+
JSON_PRMTV_TYPES: Type = Union[str, int, float, bool, None]
|
|
23
26
|
|
|
24
27
|
__all__ = [
|
|
25
28
|
'MmifObject',
|
|
26
29
|
'MmifObjectEncoder',
|
|
27
30
|
'DataList',
|
|
28
|
-
'DataDict'
|
|
31
|
+
'DataDict',
|
|
32
|
+
'JSON_PRMTV_TYPES'
|
|
29
33
|
]
|
|
30
34
|
|
|
31
35
|
|
|
@@ -77,6 +81,8 @@ class MmifObject(object):
|
|
|
77
81
|
an ID value automatically generated, based on its parent object.
|
|
78
82
|
"""
|
|
79
83
|
|
|
84
|
+
# these are the reserved names that cannot be used as attribute names, and
|
|
85
|
+
# they won't be serialized
|
|
80
86
|
reserved_names: Set[str] = {
|
|
81
87
|
'reserved_names',
|
|
82
88
|
'_unnamed_attributes',
|
|
@@ -99,6 +105,8 @@ class MmifObject(object):
|
|
|
99
105
|
self._unnamed_attributes = {}
|
|
100
106
|
if mmif_obj is not None:
|
|
101
107
|
self.deserialize(mmif_obj)
|
|
108
|
+
for method in mmif.patches[self.__class__]:
|
|
109
|
+
setattr(self, method.__name__, MethodType(method, self))
|
|
102
110
|
|
|
103
111
|
def disallow_additional_properties(self) -> None:
|
|
104
112
|
"""
|
|
@@ -419,7 +427,7 @@ class DataList(MmifObject, Generic[T]):
|
|
|
419
427
|
return self._items.__len__()
|
|
420
428
|
|
|
421
429
|
def __reversed__(self) -> Iterator[T]:
|
|
422
|
-
return reversed(
|
|
430
|
+
return reversed(self._items.values())
|
|
423
431
|
|
|
424
432
|
def __contains__(self, item) -> bool:
|
|
425
433
|
return item in self._items
|
|
@@ -440,9 +448,6 @@ class DataDict(MmifObject, Generic[T, S]):
|
|
|
440
448
|
def _serialize(self, *args, **kwargs) -> dict:
|
|
441
449
|
return super()._serialize(self._items)
|
|
442
450
|
|
|
443
|
-
def _deserialize(self, input_dict: dict) -> None:
|
|
444
|
-
raise NotImplementedError()
|
|
445
|
-
|
|
446
451
|
def get(self, key: T, default=None) -> Optional[S]:
|
|
447
452
|
return self._items.get(key, default)
|
|
448
453
|
|
|
@@ -47,8 +47,7 @@ class View(MmifObject):
|
|
|
47
47
|
self._required_attributes = ["id", "metadata", "annotations"]
|
|
48
48
|
super().__init__(view_obj)
|
|
49
49
|
for item in self.annotations:
|
|
50
|
-
|
|
51
|
-
item.parent = self.id
|
|
50
|
+
item.parent = self.id
|
|
52
51
|
|
|
53
52
|
def new_contain(self, at_type: Union[str, ThingTypesBase], **contains_metadata) -> Optional['Contain']:
|
|
54
53
|
"""
|
|
@@ -116,6 +115,7 @@ class View(MmifObject):
|
|
|
116
115
|
in the view
|
|
117
116
|
:return: the same Annotation object passed in as ``annotation``
|
|
118
117
|
"""
|
|
118
|
+
annotation.parent = self.id
|
|
119
119
|
self.annotations.append(annotation, overwrite)
|
|
120
120
|
self.new_contain(annotation.at_type)
|
|
121
121
|
return annotation
|
|
@@ -162,7 +162,6 @@ class View(MmifObject):
|
|
|
162
162
|
an existing view with the same ID
|
|
163
163
|
:return: None
|
|
164
164
|
"""
|
|
165
|
-
document.parent = self.id
|
|
166
165
|
return self.add_annotation(document, overwrite)
|
|
167
166
|
|
|
168
167
|
def get_annotations(self, at_type: Optional[Union[str, ThingTypesBase]] = None,
|
|
@@ -278,8 +277,11 @@ class ViewMetadata(MmifObject):
|
|
|
278
277
|
|
|
279
278
|
if at_type not in self.contains:
|
|
280
279
|
new_contain = Contain(contains_metadata)
|
|
281
|
-
self.
|
|
280
|
+
self.add_contain(new_contain, at_type)
|
|
282
281
|
return new_contain
|
|
282
|
+
|
|
283
|
+
def add_contain(self, contain: 'Contain', at_type: Union[str, ThingTypesBase]):
|
|
284
|
+
self.contains[at_type] = contain
|
|
283
285
|
|
|
284
286
|
def add_parameters(self, **runtime_params):
|
|
285
287
|
self.parameters.update(dict(runtime_params))
|
|
@@ -315,7 +317,7 @@ class ErrorDict(MmifObject):
|
|
|
315
317
|
super().__init__(error_obj)
|
|
316
318
|
|
|
317
319
|
|
|
318
|
-
class Contain(
|
|
320
|
+
class Contain(DataDict[str, str]):
|
|
319
321
|
"""
|
|
320
322
|
Contain object that represents the metadata of a single
|
|
321
323
|
annotation type in the ``contains`` metadata of a MMIF view.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mmif-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: Python implementation of MultiMedia Interchange Format specification. (https://mmif.clams.ai)
|
|
5
5
|
Home-page: https://mmif.clams.ai
|
|
6
6
|
Author: Brandeis Lab for Linguistics and Computation
|
mmif-python-1.0.1/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
1.0.1
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import contextlib
|
|
2
|
-
|
|
3
|
-
import pkg_resources
|
|
4
|
-
|
|
5
|
-
from mmif.ver import __version__
|
|
6
|
-
from mmif.ver import __specver__
|
|
7
|
-
from mmif.vocabulary import *
|
|
8
|
-
from mmif.serialize import *
|
|
9
|
-
|
|
10
|
-
_res_pkg = 'res'
|
|
11
|
-
_ver_pkg = 'ver'
|
|
12
|
-
_vocabulary_pkg = 'vocabulary'
|
|
13
|
-
_schema_res_name = 'mmif.json'
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def get_mmif_json_schema():
|
|
17
|
-
res = pkg_resources.resource_stream(f'{__name__}.{_res_pkg}', _schema_res_name)
|
|
18
|
-
res_str = res.read().decode('utf-8')
|
|
19
|
-
res.close()
|
|
20
|
-
return res_str
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|