ai_harmonization 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_harmonization/__init__.py +0 -0
- ai_harmonization/cde.py +644 -0
- ai_harmonization/harmonization_approaches/__init__.py +0 -0
- ai_harmonization/harmonization_approaches/base.py +202 -0
- ai_harmonization/harmonization_approaches/embeddings.py +97 -0
- ai_harmonization/harmonization_approaches/similarity_inmem.py +174 -0
- ai_harmonization/harmonization_benchmark.py +248 -0
- ai_harmonization/interactive.py +69 -0
- ai_harmonization/jsonl.py +197 -0
- ai_harmonization/main.py +12 -0
- ai_harmonization/simple_data_model.py +433 -0
- ai_harmonization/utils.py +471 -0
- ai_harmonization-0.2.6.dist-info/METADATA +159 -0
- ai_harmonization-0.2.6.dist-info/RECORD +17 -0
- ai_harmonization-0.2.6.dist-info/WHEEL +5 -0
- ai_harmonization-0.2.6.dist-info/licenses/LICENSE +201 -0
- ai_harmonization-0.2.6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import re
|
|
3
|
+
from typing import List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from langchain_core.documents import Document
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Property(BaseModel):
|
|
11
|
+
description: str
|
|
12
|
+
type: Union[str, List[str]]
|
|
13
|
+
name: str
|
|
14
|
+
additional_metadata: Optional[dict] = None
|
|
15
|
+
values: Optional[List] = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Node(BaseModel):
|
|
19
|
+
name: str
|
|
20
|
+
description: str
|
|
21
|
+
links: List[str]
|
|
22
|
+
properties: List[Property]
|
|
23
|
+
additional_metadata: Optional[dict] = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SimpleDataModel(BaseModel):
|
|
27
|
+
nodes: List[Node]
|
|
28
|
+
additional_metadata: Optional[dict] = None
|
|
29
|
+
|
|
30
|
+
def get_property_df(self) -> pd.DataFrame:
|
|
31
|
+
"""
|
|
32
|
+
Get a pandas DataFrame with a row per property from the SimpleDataModel.
|
|
33
|
+
"""
|
|
34
|
+
property_list = []
|
|
35
|
+
for node in self.nodes:
|
|
36
|
+
for property in node.properties:
|
|
37
|
+
property_list.append(
|
|
38
|
+
{
|
|
39
|
+
"node_name": node.name,
|
|
40
|
+
"property_name": property.name,
|
|
41
|
+
"property_description": property.description,
|
|
42
|
+
"property_type": property.type,
|
|
43
|
+
"additional_metadata": property.additional_metadata,
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
return pd.DataFrame(property_list)
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def from_simple_json(input_json: str, ignore_properties_with_endings=None):
|
|
50
|
+
"""
|
|
51
|
+
Converts a Simple Data Model json format to a standard format object
|
|
52
|
+
"""
|
|
53
|
+
ignore_properties_with_endings = ignore_properties_with_endings or []
|
|
54
|
+
|
|
55
|
+
input_model = json.loads(input_json)
|
|
56
|
+
|
|
57
|
+
nodes = []
|
|
58
|
+
for node_data in input_model["nodes"]:
|
|
59
|
+
properties = []
|
|
60
|
+
for prop_data in node_data["properties"]:
|
|
61
|
+
property_name = prop_data.get("name", prop_data.get("name:", ""))
|
|
62
|
+
|
|
63
|
+
skip = False
|
|
64
|
+
for ending in ignore_properties_with_endings:
|
|
65
|
+
if property_name.strip().endswith(ending):
|
|
66
|
+
skip = True
|
|
67
|
+
if skip:
|
|
68
|
+
continue
|
|
69
|
+
|
|
70
|
+
properties.append(
|
|
71
|
+
Property(
|
|
72
|
+
description=prop_data.get("description", ""),
|
|
73
|
+
type=prop_data.get("type", ""),
|
|
74
|
+
name=property_name,
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
nodes.append(
|
|
79
|
+
Node(
|
|
80
|
+
name=node_data.get("name", node_data.get("name:", "")),
|
|
81
|
+
description=node_data.get("description", ""),
|
|
82
|
+
links=node_data.get("links", []),
|
|
83
|
+
properties=properties,
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
data_model = SimpleDataModel(nodes=nodes)
|
|
88
|
+
return data_model
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def from_gen3_model(input_json: str, ignore_properties_with_endings=None):
|
|
92
|
+
"""
|
|
93
|
+
Converts a Gen3 DD JSON model to a standard format
|
|
94
|
+
"""
|
|
95
|
+
ignore_properties_with_endings = ignore_properties_with_endings or []
|
|
96
|
+
|
|
97
|
+
gen3_model = json.loads(input_json)
|
|
98
|
+
data_model = SimpleDataModel(nodes=[])
|
|
99
|
+
|
|
100
|
+
for node_name, node_info in gen3_model.items():
|
|
101
|
+
if node_name in [
|
|
102
|
+
"_terms",
|
|
103
|
+
"_settings",
|
|
104
|
+
"_definitions",
|
|
105
|
+
"metaschema",
|
|
106
|
+
"root",
|
|
107
|
+
]:
|
|
108
|
+
continue
|
|
109
|
+
|
|
110
|
+
# Convert each property in the gen3_model to our Property model
|
|
111
|
+
properties = []
|
|
112
|
+
for property_name, property_info in node_info.get("properties", {}).items():
|
|
113
|
+
skip = False
|
|
114
|
+
for ending in ignore_properties_with_endings:
|
|
115
|
+
if property_name.strip().endswith(ending):
|
|
116
|
+
skip = True
|
|
117
|
+
if skip:
|
|
118
|
+
continue
|
|
119
|
+
|
|
120
|
+
# handle foreign key links
|
|
121
|
+
if "anyOf" in property_info.keys():
|
|
122
|
+
for sub_properties in property_info["anyOf"]:
|
|
123
|
+
sub_node_name = property_name
|
|
124
|
+
|
|
125
|
+
if sub_node_name.endswith("s"):
|
|
126
|
+
sub_node_name = sub_node_name[:-1]
|
|
127
|
+
|
|
128
|
+
# TODO better handle non-plural
|
|
129
|
+
|
|
130
|
+
if sub_node_name.endswith("ies"):
|
|
131
|
+
sub_node_name = sub_node_name[:-3] + "y"
|
|
132
|
+
|
|
133
|
+
for sub_property_name, sub_property_info in (
|
|
134
|
+
sub_properties.get("items", {})
|
|
135
|
+
.get("properties", {})
|
|
136
|
+
.items()
|
|
137
|
+
):
|
|
138
|
+
skip = False
|
|
139
|
+
for ending in ignore_properties_with_endings:
|
|
140
|
+
if sub_property_name.strip().endswith(ending):
|
|
141
|
+
skip = True
|
|
142
|
+
if skip:
|
|
143
|
+
continue
|
|
144
|
+
|
|
145
|
+
node_property = Property(
|
|
146
|
+
description=sub_property_info.get("description", ""),
|
|
147
|
+
type=sub_property_info.get("type", ""),
|
|
148
|
+
name=f"{sub_node_name}.{sub_property_name}",
|
|
149
|
+
)
|
|
150
|
+
if not node_property.type:
|
|
151
|
+
if "enum" in sub_property_info:
|
|
152
|
+
node_property.type = "enum"
|
|
153
|
+
if not node_property.description:
|
|
154
|
+
if "term" in sub_property_info:
|
|
155
|
+
node_property.description = sub_property_info[
|
|
156
|
+
"term"
|
|
157
|
+
].get("description", "")
|
|
158
|
+
|
|
159
|
+
if (
|
|
160
|
+
"term" in sub_property_info
|
|
161
|
+
and "termDef" in sub_property_info["term"]
|
|
162
|
+
):
|
|
163
|
+
if not node_property.additional_metadata:
|
|
164
|
+
node_property.additional_metadata = {}
|
|
165
|
+
|
|
166
|
+
node_property.additional_metadata.update(
|
|
167
|
+
{"cde_info": sub_property_info["term"]["termDef"]}
|
|
168
|
+
)
|
|
169
|
+
properties.append(node_property)
|
|
170
|
+
else:
|
|
171
|
+
node_property = Property(
|
|
172
|
+
description=property_info.get("description", ""),
|
|
173
|
+
type=property_info.get("type", ""),
|
|
174
|
+
name=property_name,
|
|
175
|
+
)
|
|
176
|
+
if not node_property.type:
|
|
177
|
+
if "enum" in property_info:
|
|
178
|
+
node_property.type = "enum"
|
|
179
|
+
if not node_property.description:
|
|
180
|
+
if "term" in property_info:
|
|
181
|
+
node_property.description = property_info["term"].get(
|
|
182
|
+
"description", ""
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
if "term" in property_info and "termDef" in property_info["term"]:
|
|
186
|
+
if not node_property.additional_metadata:
|
|
187
|
+
node_property.additional_metadata = {}
|
|
188
|
+
|
|
189
|
+
node_property.additional_metadata.update(
|
|
190
|
+
{"cde_info": property_info["term"]["termDef"]}
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
properties.append(node_property)
|
|
194
|
+
|
|
195
|
+
node = Node(
|
|
196
|
+
name=node_name,
|
|
197
|
+
description=node_info["description"],
|
|
198
|
+
properties=properties,
|
|
199
|
+
links=[
|
|
200
|
+
link_info["name"]
|
|
201
|
+
for link_info in node_info.get("links", [])
|
|
202
|
+
if "name" in link_info
|
|
203
|
+
],
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
if not node.links and "subgroup" in node_info.get("links", {}):
|
|
207
|
+
node.links = [
|
|
208
|
+
link_info["name"] for link_info in node_info["links"]["subgroup"]
|
|
209
|
+
]
|
|
210
|
+
|
|
211
|
+
data_model.nodes.append(node)
|
|
212
|
+
|
|
213
|
+
return data_model
|
|
214
|
+
|
|
215
|
+
@staticmethod
|
|
216
|
+
def from_gdc_model(input_json: str):
|
|
217
|
+
"""
|
|
218
|
+
Converts a GDC model to a standard SimpleDataModel
|
|
219
|
+
Accepts input as a json string or a parsed dict.
|
|
220
|
+
"""
|
|
221
|
+
# If input is a string, parse it
|
|
222
|
+
if isinstance(input_json, str):
|
|
223
|
+
gdc_dict = json.loads(input_json)
|
|
224
|
+
else:
|
|
225
|
+
gdc_dict = input_json
|
|
226
|
+
|
|
227
|
+
nodes = []
|
|
228
|
+
for node_name, node_data in gdc_dict.items():
|
|
229
|
+
# Skip metadata/_definitions nodes
|
|
230
|
+
if node_name.startswith("_"):
|
|
231
|
+
continue
|
|
232
|
+
|
|
233
|
+
# Node might have description missing, so use empty string
|
|
234
|
+
node_description = node_data.get("description", "")
|
|
235
|
+
properties = []
|
|
236
|
+
|
|
237
|
+
for prop_name, prop_data in node_data.get("properties", {}).items():
|
|
238
|
+
# Exctract values
|
|
239
|
+
values = None
|
|
240
|
+
if "enum" in prop_data:
|
|
241
|
+
values = prop_data["enum"]
|
|
242
|
+
elif "values" in prop_data:
|
|
243
|
+
values = prop_data["values"]
|
|
244
|
+
|
|
245
|
+
# Build a Property object
|
|
246
|
+
prop = Property(
|
|
247
|
+
name=prop_name,
|
|
248
|
+
description=prop_data.get("description", ""),
|
|
249
|
+
type=prop_data.get("type", ""),
|
|
250
|
+
values=values,
|
|
251
|
+
)
|
|
252
|
+
properties.append(prop)
|
|
253
|
+
|
|
254
|
+
# Links: GDC rarely has links, but may add as empty list
|
|
255
|
+
links = []
|
|
256
|
+
if isinstance(node_data.get("links", []), list):
|
|
257
|
+
links = [
|
|
258
|
+
link.get("name", "")
|
|
259
|
+
for link in node_data.get("links", [])
|
|
260
|
+
if "name" in link
|
|
261
|
+
]
|
|
262
|
+
elif (
|
|
263
|
+
isinstance(node_data.get("links", {}), dict)
|
|
264
|
+
and "subgroup" in node_data["links"]
|
|
265
|
+
):
|
|
266
|
+
links = [
|
|
267
|
+
link_info["name"] for link_info in node_data["links"]["subgroup"]
|
|
268
|
+
]()
|
|
269
|
+
|
|
270
|
+
# Build Node
|
|
271
|
+
node = Node(
|
|
272
|
+
name=node_name,
|
|
273
|
+
description=node_description,
|
|
274
|
+
properties=properties,
|
|
275
|
+
links=links,
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
nodes.append(node)
|
|
279
|
+
|
|
280
|
+
data_model = SimpleDataModel(nodes=nodes)
|
|
281
|
+
return data_model
|
|
282
|
+
|
|
283
|
+
@staticmethod
|
|
284
|
+
def from_linkml_jsonschema(input_json: str, ignore_properties_with_endings=None):
|
|
285
|
+
"""
|
|
286
|
+
Converts a LinkML JSON model to a standard format
|
|
287
|
+
"""
|
|
288
|
+
ignore_properties_with_endings = ignore_properties_with_endings or []
|
|
289
|
+
|
|
290
|
+
input_model = json.loads(input_json)
|
|
291
|
+
data_model = SimpleDataModel(nodes=[])
|
|
292
|
+
|
|
293
|
+
for node_name, node_info in input_model["$defs"].items():
|
|
294
|
+
node_description: str = node_info.get("description", "")
|
|
295
|
+
properties: List[Property] = []
|
|
296
|
+
for property_name, property_info in node_info.get("properties", {}).items():
|
|
297
|
+
skip = False
|
|
298
|
+
for ending in ignore_properties_with_endings:
|
|
299
|
+
if property_name.strip().endswith(ending):
|
|
300
|
+
skip = True
|
|
301
|
+
if skip:
|
|
302
|
+
continue
|
|
303
|
+
|
|
304
|
+
property_type = property_info.get("type", "")
|
|
305
|
+
property_description = property_info.get("description", "")
|
|
306
|
+
if "type" in property_info:
|
|
307
|
+
del property_info["type"]
|
|
308
|
+
if "description" in property_info:
|
|
309
|
+
del property_info["description"]
|
|
310
|
+
properties.append(
|
|
311
|
+
Property(
|
|
312
|
+
name=property_name,
|
|
313
|
+
description=property_description,
|
|
314
|
+
type=property_type,
|
|
315
|
+
additional_metadata=property_info,
|
|
316
|
+
)
|
|
317
|
+
)
|
|
318
|
+
if "properties" in node_info:
|
|
319
|
+
del node_info["properties"]
|
|
320
|
+
|
|
321
|
+
del node_info["description"]
|
|
322
|
+
|
|
323
|
+
# TODO: populate links
|
|
324
|
+
node_links = []
|
|
325
|
+
|
|
326
|
+
data_model.nodes.append(
|
|
327
|
+
Node(
|
|
328
|
+
name=node_name,
|
|
329
|
+
properties=properties,
|
|
330
|
+
links=node_links,
|
|
331
|
+
description=node_description,
|
|
332
|
+
additional_metadata=node_info,
|
|
333
|
+
)
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
return data_model
|
|
337
|
+
|
|
338
|
+
@staticmethod
|
|
339
|
+
def get_from_unknown_json_format(input_json: str, *args, **kwargs):
|
|
340
|
+
simple_data_model = None
|
|
341
|
+
exception: BaseException = BaseException()
|
|
342
|
+
|
|
343
|
+
try:
|
|
344
|
+
simple_data_model = SimpleDataModel.from_simple_json(
|
|
345
|
+
input_json, *args, **kwargs
|
|
346
|
+
)
|
|
347
|
+
except BaseException as exc:
|
|
348
|
+
exception = exc
|
|
349
|
+
pass
|
|
350
|
+
|
|
351
|
+
try:
|
|
352
|
+
simple_data_model = SimpleDataModel.from_gen3_model(
|
|
353
|
+
input_json, *args, **kwargs
|
|
354
|
+
)
|
|
355
|
+
except BaseException as exc:
|
|
356
|
+
exception = exc
|
|
357
|
+
pass
|
|
358
|
+
|
|
359
|
+
try:
|
|
360
|
+
simple_data_model = SimpleDataModel.from_gdc_model(input_json)
|
|
361
|
+
except BaseException as exc:
|
|
362
|
+
exception = exc
|
|
363
|
+
pass
|
|
364
|
+
|
|
365
|
+
if not simple_data_model:
|
|
366
|
+
print(
|
|
367
|
+
"Could not convert from unknown format to SimpleDataModel. Consider writing a custom converter."
|
|
368
|
+
)
|
|
369
|
+
raise exception
|
|
370
|
+
|
|
371
|
+
return simple_data_model
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def get_node_prop_type_desc_from_string(input_string: str) -> tuple[str, str, str, str]:
|
|
375
|
+
"""
|
|
376
|
+
Parses a string of the format "node.property_name (type): desc" or "node.property_name: desc"
|
|
377
|
+
and returns a tuple containing the node name, property name, property type, and property description.
|
|
378
|
+
"""
|
|
379
|
+
match = re.match(r"^(.*?)\.(.*?)\s*(?:\(((?:(?!\)).)*?)\):|:)(.*)$", input_string)
|
|
380
|
+
if match:
|
|
381
|
+
node_name = match.group(1) or ""
|
|
382
|
+
prop_name = match.group(2) or ""
|
|
383
|
+
prop_type = match.group(3) or ""
|
|
384
|
+
prop_desc = match.group(4) or ""
|
|
385
|
+
return (
|
|
386
|
+
node_name.strip(),
|
|
387
|
+
prop_name.strip(),
|
|
388
|
+
prop_type.strip(),
|
|
389
|
+
prop_desc.strip(),
|
|
390
|
+
)
|
|
391
|
+
return ("", "", "", "")
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def get_data_model_as_node_prop_type_descriptions(
|
|
395
|
+
data_model: SimpleDataModel,
|
|
396
|
+
) -> List[str]:
|
|
397
|
+
"""
|
|
398
|
+
Retrieves and converts a data model output to a list of strings with the format:
|
|
399
|
+
node_name.property_name (type): property_desc
|
|
400
|
+
|
|
401
|
+
Returns:
|
|
402
|
+
List[str]: A list of strings representing the property
|
|
403
|
+
"""
|
|
404
|
+
node_prop_descs = []
|
|
405
|
+
for node in data_model.nodes:
|
|
406
|
+
for property in node.properties:
|
|
407
|
+
value = get_node_property_as_string(node, property)
|
|
408
|
+
node_prop_descs.append(value)
|
|
409
|
+
return node_prop_descs
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def get_node_property_as_string(node: Node, node_property: Property) -> str:
|
|
413
|
+
property_desc = node_property.description.replace("\t", " ").replace("\n", " ")
|
|
414
|
+
return f"{node.name}.{node_property.name} ({node_property.type}): {property_desc}"
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def get_data_model_as_langchain_documents(
|
|
418
|
+
data_model: SimpleDataModel,
|
|
419
|
+
) -> List[Document]:
|
|
420
|
+
"""
|
|
421
|
+
Retrieves and converts a data model output to LangChain documents.
|
|
422
|
+
|
|
423
|
+
Returns:
|
|
424
|
+
List[Document]: A list of Document objects representing the converted LangChain documents.
|
|
425
|
+
"""
|
|
426
|
+
documents = []
|
|
427
|
+
node_prop_descs = get_data_model_as_node_prop_type_descriptions(data_model)
|
|
428
|
+
|
|
429
|
+
for value in node_prop_descs:
|
|
430
|
+
document = Document(page_content=value, metadata={})
|
|
431
|
+
documents.append(document)
|
|
432
|
+
|
|
433
|
+
return documents
|