ai_harmonization 0.2.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,433 @@
1
+ import json
2
+ import re
3
+ from typing import List, Optional, Union
4
+
5
+ import pandas as pd
6
+ from langchain_core.documents import Document
7
+ from pydantic import BaseModel
8
+
9
+
10
+ class Property(BaseModel):
11
+ description: str
12
+ type: Union[str, List[str]]
13
+ name: str
14
+ additional_metadata: Optional[dict] = None
15
+ values: Optional[List] = None
16
+
17
+
18
+ class Node(BaseModel):
19
+ name: str
20
+ description: str
21
+ links: List[str]
22
+ properties: List[Property]
23
+ additional_metadata: Optional[dict] = None
24
+
25
+
26
+ class SimpleDataModel(BaseModel):
27
+ nodes: List[Node]
28
+ additional_metadata: Optional[dict] = None
29
+
30
+ def get_property_df(self) -> pd.DataFrame:
31
+ """
32
+ Get a pandas DataFrame with a row per property from the SimpleDataModel.
33
+ """
34
+ property_list = []
35
+ for node in self.nodes:
36
+ for property in node.properties:
37
+ property_list.append(
38
+ {
39
+ "node_name": node.name,
40
+ "property_name": property.name,
41
+ "property_description": property.description,
42
+ "property_type": property.type,
43
+ "additional_metadata": property.additional_metadata,
44
+ }
45
+ )
46
+ return pd.DataFrame(property_list)
47
+
48
+ @staticmethod
49
+ def from_simple_json(input_json: str, ignore_properties_with_endings=None):
50
+ """
51
+ Converts a Simple Data Model json format to a standard format object
52
+ """
53
+ ignore_properties_with_endings = ignore_properties_with_endings or []
54
+
55
+ input_model = json.loads(input_json)
56
+
57
+ nodes = []
58
+ for node_data in input_model["nodes"]:
59
+ properties = []
60
+ for prop_data in node_data["properties"]:
61
+ property_name = prop_data.get("name", prop_data.get("name:", ""))
62
+
63
+ skip = False
64
+ for ending in ignore_properties_with_endings:
65
+ if property_name.strip().endswith(ending):
66
+ skip = True
67
+ if skip:
68
+ continue
69
+
70
+ properties.append(
71
+ Property(
72
+ description=prop_data.get("description", ""),
73
+ type=prop_data.get("type", ""),
74
+ name=property_name,
75
+ )
76
+ )
77
+
78
+ nodes.append(
79
+ Node(
80
+ name=node_data.get("name", node_data.get("name:", "")),
81
+ description=node_data.get("description", ""),
82
+ links=node_data.get("links", []),
83
+ properties=properties,
84
+ )
85
+ )
86
+
87
+ data_model = SimpleDataModel(nodes=nodes)
88
+ return data_model
89
+
90
+ @staticmethod
91
+ def from_gen3_model(input_json: str, ignore_properties_with_endings=None):
92
+ """
93
+ Converts a Gen3 DD JSON model to a standard format
94
+ """
95
+ ignore_properties_with_endings = ignore_properties_with_endings or []
96
+
97
+ gen3_model = json.loads(input_json)
98
+ data_model = SimpleDataModel(nodes=[])
99
+
100
+ for node_name, node_info in gen3_model.items():
101
+ if node_name in [
102
+ "_terms",
103
+ "_settings",
104
+ "_definitions",
105
+ "metaschema",
106
+ "root",
107
+ ]:
108
+ continue
109
+
110
+ # Convert each property in the gen3_model to our Property model
111
+ properties = []
112
+ for property_name, property_info in node_info.get("properties", {}).items():
113
+ skip = False
114
+ for ending in ignore_properties_with_endings:
115
+ if property_name.strip().endswith(ending):
116
+ skip = True
117
+ if skip:
118
+ continue
119
+
120
+ # handle foreign key links
121
+ if "anyOf" in property_info.keys():
122
+ for sub_properties in property_info["anyOf"]:
123
+ sub_node_name = property_name
124
+
125
+ if sub_node_name.endswith("s"):
126
+ sub_node_name = sub_node_name[:-1]
127
+
128
+ # TODO better handle non-plural
129
+
130
+ if sub_node_name.endswith("ies"):
131
+ sub_node_name = sub_node_name[:-3] + "y"
132
+
133
+ for sub_property_name, sub_property_info in (
134
+ sub_properties.get("items", {})
135
+ .get("properties", {})
136
+ .items()
137
+ ):
138
+ skip = False
139
+ for ending in ignore_properties_with_endings:
140
+ if sub_property_name.strip().endswith(ending):
141
+ skip = True
142
+ if skip:
143
+ continue
144
+
145
+ node_property = Property(
146
+ description=sub_property_info.get("description", ""),
147
+ type=sub_property_info.get("type", ""),
148
+ name=f"{sub_node_name}.{sub_property_name}",
149
+ )
150
+ if not node_property.type:
151
+ if "enum" in sub_property_info:
152
+ node_property.type = "enum"
153
+ if not node_property.description:
154
+ if "term" in sub_property_info:
155
+ node_property.description = sub_property_info[
156
+ "term"
157
+ ].get("description", "")
158
+
159
+ if (
160
+ "term" in sub_property_info
161
+ and "termDef" in sub_property_info["term"]
162
+ ):
163
+ if not node_property.additional_metadata:
164
+ node_property.additional_metadata = {}
165
+
166
+ node_property.additional_metadata.update(
167
+ {"cde_info": sub_property_info["term"]["termDef"]}
168
+ )
169
+ properties.append(node_property)
170
+ else:
171
+ node_property = Property(
172
+ description=property_info.get("description", ""),
173
+ type=property_info.get("type", ""),
174
+ name=property_name,
175
+ )
176
+ if not node_property.type:
177
+ if "enum" in property_info:
178
+ node_property.type = "enum"
179
+ if not node_property.description:
180
+ if "term" in property_info:
181
+ node_property.description = property_info["term"].get(
182
+ "description", ""
183
+ )
184
+
185
+ if "term" in property_info and "termDef" in property_info["term"]:
186
+ if not node_property.additional_metadata:
187
+ node_property.additional_metadata = {}
188
+
189
+ node_property.additional_metadata.update(
190
+ {"cde_info": property_info["term"]["termDef"]}
191
+ )
192
+
193
+ properties.append(node_property)
194
+
195
+ node = Node(
196
+ name=node_name,
197
+ description=node_info["description"],
198
+ properties=properties,
199
+ links=[
200
+ link_info["name"]
201
+ for link_info in node_info.get("links", [])
202
+ if "name" in link_info
203
+ ],
204
+ )
205
+
206
+ if not node.links and "subgroup" in node_info.get("links", {}):
207
+ node.links = [
208
+ link_info["name"] for link_info in node_info["links"]["subgroup"]
209
+ ]
210
+
211
+ data_model.nodes.append(node)
212
+
213
+ return data_model
214
+
215
+ @staticmethod
216
+ def from_gdc_model(input_json: str):
217
+ """
218
+ Converts a GDC model to a standard SimpleDataModel
219
+ Accepts input as a json string or a parsed dict.
220
+ """
221
+ # If input is a string, parse it
222
+ if isinstance(input_json, str):
223
+ gdc_dict = json.loads(input_json)
224
+ else:
225
+ gdc_dict = input_json
226
+
227
+ nodes = []
228
+ for node_name, node_data in gdc_dict.items():
229
+ # Skip metadata/_definitions nodes
230
+ if node_name.startswith("_"):
231
+ continue
232
+
233
+ # Node might have description missing, so use empty string
234
+ node_description = node_data.get("description", "")
235
+ properties = []
236
+
237
+ for prop_name, prop_data in node_data.get("properties", {}).items():
238
+ # Exctract values
239
+ values = None
240
+ if "enum" in prop_data:
241
+ values = prop_data["enum"]
242
+ elif "values" in prop_data:
243
+ values = prop_data["values"]
244
+
245
+ # Build a Property object
246
+ prop = Property(
247
+ name=prop_name,
248
+ description=prop_data.get("description", ""),
249
+ type=prop_data.get("type", ""),
250
+ values=values,
251
+ )
252
+ properties.append(prop)
253
+
254
+ # Links: GDC rarely has links, but may add as empty list
255
+ links = []
256
+ if isinstance(node_data.get("links", []), list):
257
+ links = [
258
+ link.get("name", "")
259
+ for link in node_data.get("links", [])
260
+ if "name" in link
261
+ ]
262
+ elif (
263
+ isinstance(node_data.get("links", {}), dict)
264
+ and "subgroup" in node_data["links"]
265
+ ):
266
+ links = [
267
+ link_info["name"] for link_info in node_data["links"]["subgroup"]
268
+ ]()
269
+
270
+ # Build Node
271
+ node = Node(
272
+ name=node_name,
273
+ description=node_description,
274
+ properties=properties,
275
+ links=links,
276
+ )
277
+
278
+ nodes.append(node)
279
+
280
+ data_model = SimpleDataModel(nodes=nodes)
281
+ return data_model
282
+
283
+ @staticmethod
284
+ def from_linkml_jsonschema(input_json: str, ignore_properties_with_endings=None):
285
+ """
286
+ Converts a LinkML JSON model to a standard format
287
+ """
288
+ ignore_properties_with_endings = ignore_properties_with_endings or []
289
+
290
+ input_model = json.loads(input_json)
291
+ data_model = SimpleDataModel(nodes=[])
292
+
293
+ for node_name, node_info in input_model["$defs"].items():
294
+ node_description: str = node_info.get("description", "")
295
+ properties: List[Property] = []
296
+ for property_name, property_info in node_info.get("properties", {}).items():
297
+ skip = False
298
+ for ending in ignore_properties_with_endings:
299
+ if property_name.strip().endswith(ending):
300
+ skip = True
301
+ if skip:
302
+ continue
303
+
304
+ property_type = property_info.get("type", "")
305
+ property_description = property_info.get("description", "")
306
+ if "type" in property_info:
307
+ del property_info["type"]
308
+ if "description" in property_info:
309
+ del property_info["description"]
310
+ properties.append(
311
+ Property(
312
+ name=property_name,
313
+ description=property_description,
314
+ type=property_type,
315
+ additional_metadata=property_info,
316
+ )
317
+ )
318
+ if "properties" in node_info:
319
+ del node_info["properties"]
320
+
321
+ del node_info["description"]
322
+
323
+ # TODO: populate links
324
+ node_links = []
325
+
326
+ data_model.nodes.append(
327
+ Node(
328
+ name=node_name,
329
+ properties=properties,
330
+ links=node_links,
331
+ description=node_description,
332
+ additional_metadata=node_info,
333
+ )
334
+ )
335
+
336
+ return data_model
337
+
338
+ @staticmethod
339
+ def get_from_unknown_json_format(input_json: str, *args, **kwargs):
340
+ simple_data_model = None
341
+ exception: BaseException = BaseException()
342
+
343
+ try:
344
+ simple_data_model = SimpleDataModel.from_simple_json(
345
+ input_json, *args, **kwargs
346
+ )
347
+ except BaseException as exc:
348
+ exception = exc
349
+ pass
350
+
351
+ try:
352
+ simple_data_model = SimpleDataModel.from_gen3_model(
353
+ input_json, *args, **kwargs
354
+ )
355
+ except BaseException as exc:
356
+ exception = exc
357
+ pass
358
+
359
+ try:
360
+ simple_data_model = SimpleDataModel.from_gdc_model(input_json)
361
+ except BaseException as exc:
362
+ exception = exc
363
+ pass
364
+
365
+ if not simple_data_model:
366
+ print(
367
+ "Could not convert from unknown format to SimpleDataModel. Consider writing a custom converter."
368
+ )
369
+ raise exception
370
+
371
+ return simple_data_model
372
+
373
+
374
+ def get_node_prop_type_desc_from_string(input_string: str) -> tuple[str, str, str, str]:
375
+ """
376
+ Parses a string of the format "node.property_name (type): desc" or "node.property_name: desc"
377
+ and returns a tuple containing the node name, property name, property type, and property description.
378
+ """
379
+ match = re.match(r"^(.*?)\.(.*?)\s*(?:\(((?:(?!\)).)*?)\):|:)(.*)$", input_string)
380
+ if match:
381
+ node_name = match.group(1) or ""
382
+ prop_name = match.group(2) or ""
383
+ prop_type = match.group(3) or ""
384
+ prop_desc = match.group(4) or ""
385
+ return (
386
+ node_name.strip(),
387
+ prop_name.strip(),
388
+ prop_type.strip(),
389
+ prop_desc.strip(),
390
+ )
391
+ return ("", "", "", "")
392
+
393
+
394
+ def get_data_model_as_node_prop_type_descriptions(
395
+ data_model: SimpleDataModel,
396
+ ) -> List[str]:
397
+ """
398
+ Retrieves and converts a data model output to a list of strings with the format:
399
+ node_name.property_name (type): property_desc
400
+
401
+ Returns:
402
+ List[str]: A list of strings representing the property
403
+ """
404
+ node_prop_descs = []
405
+ for node in data_model.nodes:
406
+ for property in node.properties:
407
+ value = get_node_property_as_string(node, property)
408
+ node_prop_descs.append(value)
409
+ return node_prop_descs
410
+
411
+
412
+ def get_node_property_as_string(node: Node, node_property: Property) -> str:
413
+ property_desc = node_property.description.replace("\t", " ").replace("\n", " ")
414
+ return f"{node.name}.{node_property.name} ({node_property.type}): {property_desc}"
415
+
416
+
417
+ def get_data_model_as_langchain_documents(
418
+ data_model: SimpleDataModel,
419
+ ) -> List[Document]:
420
+ """
421
+ Retrieves and converts a data model output to LangChain documents.
422
+
423
+ Returns:
424
+ List[Document]: A list of Document objects representing the converted LangChain documents.
425
+ """
426
+ documents = []
427
+ node_prop_descs = get_data_model_as_node_prop_type_descriptions(data_model)
428
+
429
+ for value in node_prop_descs:
430
+ document = Document(page_content=value, metadata={})
431
+ documents.append(document)
432
+
433
+ return documents