ai_harmonization 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_harmonization/__init__.py +0 -0
- ai_harmonization/cde.py +644 -0
- ai_harmonization/harmonization_approaches/__init__.py +0 -0
- ai_harmonization/harmonization_approaches/base.py +202 -0
- ai_harmonization/harmonization_approaches/embeddings.py +97 -0
- ai_harmonization/harmonization_approaches/similarity_inmem.py +174 -0
- ai_harmonization/harmonization_benchmark.py +248 -0
- ai_harmonization/interactive.py +69 -0
- ai_harmonization/jsonl.py +197 -0
- ai_harmonization/main.py +12 -0
- ai_harmonization/simple_data_model.py +433 -0
- ai_harmonization/utils.py +471 -0
- ai_harmonization-0.2.6.dist-info/METADATA +159 -0
- ai_harmonization-0.2.6.dist-info/RECORD +17 -0
- ai_harmonization-0.2.6.dist-info/WHEEL +5 -0
- ai_harmonization-0.2.6.dist-info/licenses/LICENSE +201 -0
- ai_harmonization-0.2.6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
import bz2
|
|
2
|
+
import csv
|
|
3
|
+
import gzip
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import shutil
|
|
9
|
+
import tarfile
|
|
10
|
+
import warnings
|
|
11
|
+
import zipfile
|
|
12
|
+
from csv import DictReader
|
|
13
|
+
from typing import List, Optional, Tuple, Union
|
|
14
|
+
|
|
15
|
+
import chromadb
|
|
16
|
+
import requests
|
|
17
|
+
from chromadb.api.types import Documents, Embeddings, IDs, Metadatas
|
|
18
|
+
from langchain_chroma import Chroma
|
|
19
|
+
from langchain_core.documents import Document
|
|
20
|
+
from langchain_core.embeddings import Embeddings
|
|
21
|
+
from langchain_huggingface import HuggingFaceEmbeddings
|
|
22
|
+
from tqdm import tqdm
|
|
23
|
+
|
|
24
|
+
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
25
|
+
TEMP_DIR = os.path.abspath(f"{CURRENT_DIR}/../output/temp")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def read_in_json(file_location):
|
|
29
|
+
try:
|
|
30
|
+
with open(file_location, "r") as f:
|
|
31
|
+
data = json.load(f)
|
|
32
|
+
return data, None
|
|
33
|
+
except Exception as e:
|
|
34
|
+
return None, e
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def make_dir(dirname):
|
|
38
|
+
logging.info(f"outputting to: {dirname}")
|
|
39
|
+
try:
|
|
40
|
+
os.makedirs(dirname, exist_ok=True)
|
|
41
|
+
except Exception as e:
|
|
42
|
+
logging.info("failed to make dir")
|
|
43
|
+
logging.info(str(e))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def unzip_files(directory: str, remove_compressed_file: bool = True) -> None:
|
|
47
|
+
"""
|
|
48
|
+
Unzip various types of compressed files (.gz, .tar, .tar.gz, .zip, .bz2)
|
|
49
|
+
in the specified directory and its subdirectories.
|
|
50
|
+
|
|
51
|
+
Parameters:
|
|
52
|
+
directory (str): The path to the directory to search for compressed files.
|
|
53
|
+
remove_compressed_file (bool): Whether to remove the compressed file after unzipping.
|
|
54
|
+
"""
|
|
55
|
+
# Iterate through all files in the specified directory
|
|
56
|
+
for root, dirs, files in os.walk(directory):
|
|
57
|
+
for file in files:
|
|
58
|
+
compressed_file_path = os.path.join(root, file)
|
|
59
|
+
is_compressed_file = True
|
|
60
|
+
|
|
61
|
+
if file.endswith(".gz") and not file.endswith(".tar.gz"):
|
|
62
|
+
# Handle .gz files
|
|
63
|
+
output_file_path = os.path.splitext(compressed_file_path)[
|
|
64
|
+
0
|
|
65
|
+
] # Remove .gz extension
|
|
66
|
+
with gzip.open(compressed_file_path, "rb") as f_in:
|
|
67
|
+
with open(output_file_path, "wb") as f_out:
|
|
68
|
+
shutil.copyfileobj(f_in, f_out)
|
|
69
|
+
|
|
70
|
+
logging.info(f"Unzipped: {compressed_file_path} to {output_file_path}")
|
|
71
|
+
|
|
72
|
+
elif file.endswith(".tar"):
|
|
73
|
+
# Handle .tar files
|
|
74
|
+
with tarfile.open(compressed_file_path, "r") as tar:
|
|
75
|
+
tar.extractall(
|
|
76
|
+
path=root
|
|
77
|
+
) # Extract all files into the same directory as the .tar
|
|
78
|
+
|
|
79
|
+
logging.info(f"Extracted: {compressed_file_path} to {root}")
|
|
80
|
+
|
|
81
|
+
elif file.endswith(".tar.gz") or file.endswith(".tgz"):
|
|
82
|
+
# Handle .tar.gz files
|
|
83
|
+
with tarfile.open(compressed_file_path, "r:gz") as tar:
|
|
84
|
+
tar.extractall(
|
|
85
|
+
path=root
|
|
86
|
+
) # Extract all files into the same directory as the .tar.gz
|
|
87
|
+
|
|
88
|
+
logging.info(f"Extracted: {compressed_file_path} to {root}")
|
|
89
|
+
|
|
90
|
+
elif file.endswith(".zip"):
|
|
91
|
+
# Handle .zip files
|
|
92
|
+
with zipfile.ZipFile(compressed_file_path, "r") as zip_ref:
|
|
93
|
+
zip_ref.extractall(
|
|
94
|
+
root
|
|
95
|
+
) # Extract all files into the same directory as the .zip
|
|
96
|
+
|
|
97
|
+
logging.info(f"Extracted: {compressed_file_path} to {root}")
|
|
98
|
+
|
|
99
|
+
elif file.endswith(".bz2"):
|
|
100
|
+
# Handle .bz2 files
|
|
101
|
+
output_file_path = os.path.splitext(compressed_file_path)[
|
|
102
|
+
0
|
|
103
|
+
] # Remove .bz2 extension
|
|
104
|
+
with bz2.open(compressed_file_path, "rb") as f_in:
|
|
105
|
+
with open(output_file_path, "wb") as f_out:
|
|
106
|
+
shutil.copyfileobj(f_in, f_out)
|
|
107
|
+
|
|
108
|
+
logging.info(f"Unzipped: {compressed_file_path} to {output_file_path}")
|
|
109
|
+
|
|
110
|
+
else:
|
|
111
|
+
is_compressed_file = False
|
|
112
|
+
|
|
113
|
+
# Remove the compressed file if specified
|
|
114
|
+
if is_compressed_file and remove_compressed_file:
|
|
115
|
+
logging.info(f"Removing: {compressed_file_path}")
|
|
116
|
+
os.remove(compressed_file_path)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def remove_unwanted_files(
|
|
120
|
+
directory: str, inclusion_list: list[str], exclusion_list: list[str]
|
|
121
|
+
) -> None:
|
|
122
|
+
"""
|
|
123
|
+
Walks through all folders and files in the given directory,
|
|
124
|
+
and removes files that don't follow the inclusion and exclusion lists.
|
|
125
|
+
|
|
126
|
+
Parameters:
|
|
127
|
+
directory (str): The path to the root directory.
|
|
128
|
+
inclusion_list (list[str]): List of allowed file extensions to keep.
|
|
129
|
+
exclusion_list (list[str]): List of file extensions to explicitly remove.
|
|
130
|
+
"""
|
|
131
|
+
for root, dirs, files in os.walk(directory):
|
|
132
|
+
for file in files:
|
|
133
|
+
file_path = os.path.join(root, file)
|
|
134
|
+
|
|
135
|
+
# Check file extension
|
|
136
|
+
_, ext = os.path.splitext(file)
|
|
137
|
+
if file.endswith(tuple(exclusion_list)):
|
|
138
|
+
# Remove if it's in the exclusion list
|
|
139
|
+
print(f"Removing excluded file: {file_path}")
|
|
140
|
+
os.remove(file_path)
|
|
141
|
+
elif ext not in inclusion_list:
|
|
142
|
+
# Remove if it's not in the inclusion list
|
|
143
|
+
print(f"Removing unwanted file: {file_path}")
|
|
144
|
+
os.remove(file_path)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def generate_markdown_from_directory_with_custom_metrics(
|
|
148
|
+
directory, prefix="custom_metrics_"
|
|
149
|
+
):
|
|
150
|
+
"""
|
|
151
|
+
Generate markdown files from a directory of CSV files with Custom Metrics.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
directory (str): The path to the directory containing the CSV files. Defaults to None.
|
|
155
|
+
prefix (str, optional): The prefix for the CSV file names. Defaults to "custom_metrics_".
|
|
156
|
+
|
|
157
|
+
Returns:
|
|
158
|
+
None
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
if directory is not None and not os.path.exists(directory):
|
|
162
|
+
logging.error(f"Directory '{directory}' does not exist.")
|
|
163
|
+
return
|
|
164
|
+
|
|
165
|
+
output_dir = os.path.abspath(os.path.join(os.getcwd(), "markdown"))
|
|
166
|
+
if directory is not None:
|
|
167
|
+
output_dir = os.path.abspath(os.path.join(directory, "markdown"))
|
|
168
|
+
|
|
169
|
+
# Create the output directory if it doesn't exist
|
|
170
|
+
if not os.path.exists(output_dir):
|
|
171
|
+
os.makedirs(output_dir)
|
|
172
|
+
|
|
173
|
+
for root, dirs, files in os.walk(directory):
|
|
174
|
+
for file in files:
|
|
175
|
+
if (
|
|
176
|
+
file.endswith(".csv")
|
|
177
|
+
and "custom_metrics_" in file
|
|
178
|
+
and ".ipynb_checkpoints" not in file
|
|
179
|
+
):
|
|
180
|
+
file_path = os.path.join(root, file)
|
|
181
|
+
# Read the CSV file into a DictReader object
|
|
182
|
+
with open(file_path, "r") as input_file:
|
|
183
|
+
reader = DictReader(input_file)
|
|
184
|
+
|
|
185
|
+
# Iterate over each row in the reader
|
|
186
|
+
for index, row in enumerate(reader):
|
|
187
|
+
metrics_json = json.loads(
|
|
188
|
+
row["custom_metrics"].replace('""', '"').replace("'", '"')
|
|
189
|
+
)
|
|
190
|
+
metrics_json = {
|
|
191
|
+
key: value
|
|
192
|
+
for key, value in metrics_json["accuracy"].items()
|
|
193
|
+
if key.startswith("_")
|
|
194
|
+
}
|
|
195
|
+
info_to_write = {
|
|
196
|
+
"metrics": json.dumps(metrics_json, indent=2),
|
|
197
|
+
"input": row["input_text"].replace('""', '"'),
|
|
198
|
+
"output": row["generated_output_text"].replace('""', '"'),
|
|
199
|
+
"expected_output": row["output_text"].replace('""', '"'),
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
# Create the output file path
|
|
203
|
+
output_file_path = (
|
|
204
|
+
file_path.replace(".csv", "") + f"/line_{index}.md"
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
|
|
208
|
+
|
|
209
|
+
# Write the row to a new CSV file
|
|
210
|
+
with open(output_file_path, "w") as output_file:
|
|
211
|
+
output_file.write("Metrics:\n" + info_to_write["metrics"])
|
|
212
|
+
output_file.write("\n\nInput:\n" + info_to_write["input"])
|
|
213
|
+
output_file.write("\n\nAI Model Output:\n")
|
|
214
|
+
if True: # wrap_model_output_in_json_codeblock
|
|
215
|
+
output_file.write("```json\n")
|
|
216
|
+
output_file.write(info_to_write["output"])
|
|
217
|
+
if True: # wrap_model_output_in_json_codeblock
|
|
218
|
+
output_file.write("\n```")
|
|
219
|
+
|
|
220
|
+
output_file.write(
|
|
221
|
+
"\n\nExpected Output:\n```json\n"
|
|
222
|
+
+ info_to_write["expected_output"]
|
|
223
|
+
+ "\n```"
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def write_to_csv(filepath: str, data, header=None):
|
|
228
|
+
"""Writes a CSV file to the given filepath, creating directories if needed.
|
|
229
|
+
|
|
230
|
+
Args:
|
|
231
|
+
filepath (str): The full path of the CSV file.
|
|
232
|
+
data (list of lists): The data to write.
|
|
233
|
+
header (list, optional): Column headers for the CSV.
|
|
234
|
+
"""
|
|
235
|
+
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
236
|
+
|
|
237
|
+
with open(filepath, mode="w", newline="") as f:
|
|
238
|
+
writer = csv.writer(f)
|
|
239
|
+
if header:
|
|
240
|
+
writer.writerow(header)
|
|
241
|
+
writer.writerows(data)
|
|
242
|
+
return True
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def nodeproperty_to_curie(s: str) -> str:
|
|
246
|
+
"""
|
|
247
|
+
Converts a 'node.property' string to a CURIE with EX: prefix, e.g.
|
|
248
|
+
'MedicalEncounterRecord.Molecular Evidence Present' ->
|
|
249
|
+
'EX:MedicalEncounterRecord_Molecular_Evidence_Present'
|
|
250
|
+
If the input does not contain a dot, treats the whole string as 'node'.
|
|
251
|
+
"""
|
|
252
|
+
if "." in s:
|
|
253
|
+
node, prop = s.split(".", 1)
|
|
254
|
+
else:
|
|
255
|
+
node, prop = s, ""
|
|
256
|
+
|
|
257
|
+
def munge(t):
|
|
258
|
+
t = t.strip()
|
|
259
|
+
# Convert all non-alphanumeric characters to underscores
|
|
260
|
+
t = re.sub(r"\W+", "_", t)
|
|
261
|
+
# Remove leading/trailing underscores
|
|
262
|
+
t = t.strip("_")
|
|
263
|
+
return t
|
|
264
|
+
|
|
265
|
+
node_str = munge(node)
|
|
266
|
+
prop_str = munge(prop)
|
|
267
|
+
if prop_str:
|
|
268
|
+
return f"EX:{node_str}_{prop_str}"
|
|
269
|
+
else:
|
|
270
|
+
return f"EX:{node_str}"
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def get_gen3_json_schemas_and_templates(
|
|
274
|
+
url: str, output_dir: str, internal_folder_name: str = "Unmodified"
|
|
275
|
+
) -> None:
|
|
276
|
+
"""
|
|
277
|
+
Fetches JSON schema files and templates from a given Gen3 API URL and saves them to the specified directory.
|
|
278
|
+
|
|
279
|
+
Args:
|
|
280
|
+
url (str): The base URL of the API endpoint.
|
|
281
|
+
output_dir (str): The directory where the fetched data will be saved.
|
|
282
|
+
internal_folder_name (str, optional): The name of the internal folder (which is within the {{url}} folder. Defaults to "Unmodified".
|
|
283
|
+
Returns:
|
|
284
|
+
None
|
|
285
|
+
"""
|
|
286
|
+
# Construct full directory path
|
|
287
|
+
output_dir = (
|
|
288
|
+
output_dir.rstrip("/") + f"/{url.split('://')[-1]}" + f"/{internal_folder_name}"
|
|
289
|
+
)
|
|
290
|
+
try:
|
|
291
|
+
all_schemas_url = url.rstrip("/") + "/api/v0/submission/_dictionary/_all"
|
|
292
|
+
# Fetch data from the API
|
|
293
|
+
all_schemas_response = requests.get(all_schemas_url)
|
|
294
|
+
all_schemas_data = all_schemas_response.json()
|
|
295
|
+
# Define a directory to save JSON schema files
|
|
296
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
297
|
+
os.makedirs(f"{output_dir}/nodes/", exist_ok=True)
|
|
298
|
+
total_schemas = len(all_schemas_data)
|
|
299
|
+
|
|
300
|
+
schema_filename = f"{output_dir}/schema.json"
|
|
301
|
+
with open(schema_filename, "w") as file:
|
|
302
|
+
json.dump(all_schemas_data, file, indent=4)
|
|
303
|
+
|
|
304
|
+
for key, value in tqdm(
|
|
305
|
+
all_schemas_data.items(),
|
|
306
|
+
total=total_schemas,
|
|
307
|
+
desc=f"Getting Gen3 JSON Schemas from {all_schemas_url}:",
|
|
308
|
+
):
|
|
309
|
+
schema_filename = f"{output_dir}/nodes/{key}_schema.json"
|
|
310
|
+
with open(schema_filename, "w") as file:
|
|
311
|
+
json.dump(value, file, indent=4)
|
|
312
|
+
# Get the template for this item and output in the same dir
|
|
313
|
+
template_url = url.rstrip("/") + f"/api/v0/submission/template/{key}"
|
|
314
|
+
template_response = requests.get(template_url)
|
|
315
|
+
if template_response.status_code == 200:
|
|
316
|
+
template_filename = f"{output_dir}/nodes/{key}_template.tsv"
|
|
317
|
+
with open(template_filename, "w") as file:
|
|
318
|
+
file.write(template_response.text)
|
|
319
|
+
logging.info(
|
|
320
|
+
f"All JSON schemas and templates for {url} are saved here: {output_dir}."
|
|
321
|
+
)
|
|
322
|
+
except Exception as e:
|
|
323
|
+
logging.error(f"An error occurred: {e}")
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def create_batches(
|
|
327
|
+
batch_size: int,
|
|
328
|
+
ids: IDs,
|
|
329
|
+
embeddings: Optional[Embeddings] = None,
|
|
330
|
+
metadatas: Optional[Metadatas] = None,
|
|
331
|
+
documents: Optional[Documents] = None,
|
|
332
|
+
) -> List[Tuple[IDs, Embeddings, Optional[Metadatas], Optional[Documents]]]:
|
|
333
|
+
"""
|
|
334
|
+
Returns batches of provided batch_size from the lists of ids, embeddings, metadatas and documents.
|
|
335
|
+
|
|
336
|
+
Mimics chromadb.utils.batch_utils.create_batches behaviour but instead of using api.get_max_batch_size() for the batch creation it uses provided batch_size parameter.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
ids (IDs): list of document IDs
|
|
340
|
+
embeddings ([Embeddings]): optional list of embeddings,
|
|
341
|
+
metadatas ([Metadatas]): optional list of metadatas,
|
|
342
|
+
documents ([Documents]): optional list of documents
|
|
343
|
+
|
|
344
|
+
Returns:
|
|
345
|
+
List[Tuple[IDs, Embeddings, Optional[Metadatas], Optional[Documents]]]: list of batches
|
|
346
|
+
"""
|
|
347
|
+
_batches: List[Tuple[IDs, Embeddings, Optional[Metadatas], Optional[Documents]]] = (
|
|
348
|
+
[]
|
|
349
|
+
)
|
|
350
|
+
if len(ids) > batch_size:
|
|
351
|
+
# create split batches
|
|
352
|
+
for i in range(0, len(ids), batch_size):
|
|
353
|
+
_batches.append(
|
|
354
|
+
(
|
|
355
|
+
ids[i : i + batch_size],
|
|
356
|
+
embeddings[i : i + batch_size] if embeddings else None,
|
|
357
|
+
metadatas[i : i + batch_size] if metadatas else None,
|
|
358
|
+
documents[i : i + batch_size] if documents else None,
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
else:
|
|
362
|
+
_batches.append((ids, embeddings, metadatas, documents))
|
|
363
|
+
return _batches
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def add_documents_to_vectorstore(
|
|
367
|
+
documents, vectorstore, persistent_client, batch_size=None
|
|
368
|
+
):
|
|
369
|
+
batch_size = batch_size or persistent_client.get_max_batch_size()
|
|
370
|
+
logging.info(
|
|
371
|
+
"Number of documents that can be inserted at once:",
|
|
372
|
+
batch_size,
|
|
373
|
+
)
|
|
374
|
+
ids = range(len(documents))
|
|
375
|
+
batches = create_batches(
|
|
376
|
+
batch_size=batch_size, ids=list(ids), documents=list(documents)
|
|
377
|
+
)
|
|
378
|
+
for batch in batches:
|
|
379
|
+
logging.info(f"Adding batch of size {len(batch[0])}")
|
|
380
|
+
vectorstore.add_documents(documents=batch[3])
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def get_similar_documents(vectorstore, query, **kwargs) -> List[Document]:
|
|
384
|
+
"""
|
|
385
|
+
Retrieves similar documents from a vector store using a given query.
|
|
386
|
+
|
|
387
|
+
This function uses the `vectorstore.similarity_search_with_relevance_scores` method to find similar documents.
|
|
388
|
+
The returned list of documents includes their relevance scores, which can be used for ranking or filtering purposes.
|
|
389
|
+
|
|
390
|
+
Args:
|
|
391
|
+
vectorstore (Chroma): The Chroma vector store to search in.
|
|
392
|
+
query (str): The search query to use when retrieving similar documents.
|
|
393
|
+
**kwargs: Optional keyword arguments to pass to the `vectorstore.similarity_search_with_relevance_scores` method.
|
|
394
|
+
|
|
395
|
+
Returns:
|
|
396
|
+
List[Document]: A list of Document objects representing the retrieved similar documents, along with their relevance scores.
|
|
397
|
+
"""
|
|
398
|
+
# We want to ignore the printed warning when there's no documents.
|
|
399
|
+
# We don't want that in stdout polluting it, we just will have an empty list
|
|
400
|
+
with warnings.catch_warnings():
|
|
401
|
+
warnings.simplefilter("ignore")
|
|
402
|
+
return vectorstore.similarity_search_with_relevance_scores(query, **kwargs)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def get_langchain_vectorstore_and_persistent_client(
|
|
406
|
+
persist_directory_name: str = "cdes",
|
|
407
|
+
embedding_function: Union[Embeddings, None] = None,
|
|
408
|
+
chromadb_settings: Union[chromadb.config.Settings, None] = None,
|
|
409
|
+
) -> Chroma:
|
|
410
|
+
"""
|
|
411
|
+
Creates and returns a Chroma vectorstore for the 'cdes' topic.
|
|
412
|
+
|
|
413
|
+
Parameters:
|
|
414
|
+
persist_directory_name (str): The name of the persistent directory.
|
|
415
|
+
embedding_function (Embeddings | None): The language embedding function. Defaults to HuggingFaceEmbeddings with the "all-MiniLM-L6-v2" model.
|
|
416
|
+
chromadb_settings (chromadb.config.Settings | None): ChromaDB settings. Defaults to chromadb Settings with migrations_hash_algorithm="sha256" and anonymized_telemetry=False.
|
|
417
|
+
|
|
418
|
+
Returns:
|
|
419
|
+
Chroma: A Chroma vectorstore object.
|
|
420
|
+
"""
|
|
421
|
+
# We're using a general purpose language embedding algorithm
|
|
422
|
+
embedding_function = embedding_function or HuggingFaceEmbeddings(
|
|
423
|
+
model_name="all-MiniLM-L6-v2"
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
settings = chromadb_settings or chromadb.Settings(
|
|
427
|
+
migrations_hash_algorithm="sha256",
|
|
428
|
+
anonymized_telemetry=False,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
persistent_client = chromadb.PersistentClient(
|
|
432
|
+
path=f"{TEMP_DIR}/vectorstore/{persist_directory_name}", settings=settings
|
|
433
|
+
)
|
|
434
|
+
vectorstore = Chroma(
|
|
435
|
+
client=persistent_client,
|
|
436
|
+
collection_name=persist_directory_name,
|
|
437
|
+
embedding_function=embedding_function,
|
|
438
|
+
# https://docs.trychroma.com/usage-guide#changing-the-distance-function
|
|
439
|
+
collection_metadata={"hnsw:space": "cosine"},
|
|
440
|
+
persist_directory=f"{TEMP_DIR}/vectorstore/{persist_directory_name}",
|
|
441
|
+
client_settings=settings,
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
return vectorstore, persistent_client
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def get_similar_documents_as_string(documents: tuple[Document, float]) -> str:
|
|
448
|
+
"""
|
|
449
|
+
This function takes a tuple of Document objects and their corresponding similarity scores,
|
|
450
|
+
and returns a formatted string representation of the input documents.
|
|
451
|
+
|
|
452
|
+
The returned string includes the document's metadata, which is converted using pprint.pformat() for readability.
|
|
453
|
+
|
|
454
|
+
Args:
|
|
455
|
+
documents (tuple[Document, float]): A tuple containing a Document object and its corresponding similarity score.
|
|
456
|
+
|
|
457
|
+
Returns:
|
|
458
|
+
str: A formatted string representation of the input documents.
|
|
459
|
+
"""
|
|
460
|
+
output = ""
|
|
461
|
+
for doc, similarity_score in documents:
|
|
462
|
+
output += f"-" * 80 + "\n"
|
|
463
|
+
output += f"similarity_score: {similarity_score}" + "\n"
|
|
464
|
+
output += f" ids: {doc.metadata['ids']}" + "\n"
|
|
465
|
+
output += f" names: {doc.metadata['names']}" + "\n"
|
|
466
|
+
output += f" source: {doc.metadata['source']}" + "\n"
|
|
467
|
+
output += f" nihEndorsed: {doc.metadata['nihEndorsed']}" + "\n"
|
|
468
|
+
output += f" valueDomain: {doc.metadata['valueDomain']}" + "\n"
|
|
469
|
+
output += f"\n full metadata:\n"
|
|
470
|
+
output += f"{pprint.pformat(doc.metadata)}" + "\n"
|
|
471
|
+
return output
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ai_harmonization
|
|
3
|
+
Version: 0.2.6
|
|
4
|
+
Summary: AI-backed harmonization framework
|
|
5
|
+
Requires-Python: <4.0.0,>=3.13
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: chromadb>=1.0.15
|
|
9
|
+
Requires-Dist: langchain-chroma>=0.2.5
|
|
10
|
+
Requires-Dist: langchain-community>=0.3.27
|
|
11
|
+
Requires-Dist: langchain-core>=0.3.72
|
|
12
|
+
Requires-Dist: langchain-huggingface>=0.3.1
|
|
13
|
+
Requires-Dist: matplotlib>=3.10.5
|
|
14
|
+
Requires-Dist: numpy>=2.3.2
|
|
15
|
+
Requires-Dist: pandas>=2.3.1
|
|
16
|
+
Requires-Dist: pydantic>=2.11.7
|
|
17
|
+
Requires-Dist: requests>=2.32.4
|
|
18
|
+
Requires-Dist: sssom>=0.4.16
|
|
19
|
+
Requires-Dist: sssom-schema>=1.0.0
|
|
20
|
+
Requires-Dist: torch>=2.9.0
|
|
21
|
+
Requires-Dist: tqdm>=4.67.1
|
|
22
|
+
Requires-Dist: transformers>=4.57.1
|
|
23
|
+
Requires-Dist: itables>=2.4.4
|
|
24
|
+
Requires-Dist: ipython>=9.4.0
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# AI Harmonization
|
|
28
|
+
|
|
29
|
+
This contains code and related artifacts for powering an AI-assisted data model harmonization tool. It also contains the infrastructure for abstracting approaches, benchmarking them, and generating training data.
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
### **AI-Assisted Data Curation Toolkit**
|
|
34
|
+
|
|
35
|
+
- **[Demo](./jupyter/ai_assisted_data_curation.ipynb)**
|
|
36
|
+
|
|
37
|
+
### Harmonization Benchmarking and AI Training
|
|
38
|
+
|
|
39
|
+
- [Example of Evaluating a Harmonization Approach](./jupyter/harmonization_approach_evaluation.ipynb)
|
|
40
|
+
- [Example Creation of a Synthetic Benchmark for Evaluation](./jupyter/harmonization_synth_benchmark_creation.ipynb)
|
|
41
|
+
- [Example Creation of a Real Benchmark for Evaluation](./jupyter/harmonization_real_benchmark_creation.ipynb)
|
|
42
|
+
- [Example Creation of a Training Data for Harmonization](./jupyter/harmonization_training_data.ipynb)
|
|
43
|
+
|
|
44
|
+
## Details
|
|
45
|
+
|
|
46
|
+
Reference `harmonization.harmonization_approaches.similarity_inmem.SimilaritySearchInMemoryVectorDb` to understand how to build new harmonization approaches.
|
|
47
|
+
|
|
48
|
+
Abtractions have been built to allow the evaluation of different overall approaches. There is a base class to implement for a new `HarmonizationApproach`.
|
|
49
|
+
|
|
50
|
+
There are 2 Pydantic data types defined to standardize the interface. `HarmonizationSuggestions` is:
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{
|
|
54
|
+
"suggestions": List of `SingleHarmonizationSuggestion`
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
and `SingleHarmonizationSuggestion` is:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
class SingleHarmonizationSuggestion(BaseModel):
|
|
62
|
+
source_node: str
|
|
63
|
+
source_property: str
|
|
64
|
+
source_description: str
|
|
65
|
+
target_node: str
|
|
66
|
+
target_property: str
|
|
67
|
+
target_description: str
|
|
68
|
+
similarity: float = None
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
This format is convertable to [A Simple Standard for Sharing Ontological Mappings (SSSOM)](https://github.com/mapping-commons/SSSOM) and happens by default when calling `get_metrics_for_approach`.
|
|
72
|
+
|
|
73
|
+
An example of how to implement a new approach. Create a new file: `harmonization/harmonization_approaches/new_approach.py`
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from ai_curation.harmonization_approaches.harmonization import (
|
|
77
|
+
HarmonizationApproach,
|
|
78
|
+
HarmonizationSuggestions,
|
|
79
|
+
SingleHarmonizationSuggestion,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
class NewApproachExample(HarmonizationApproach):
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
):
|
|
87
|
+
super().__init__()
|
|
88
|
+
# TODO
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def get_harmonization_suggestions(
|
|
92
|
+
self, input_source_model, input_target_model, **kwargs
|
|
93
|
+
):
|
|
94
|
+
# TODO
|
|
95
|
+
return HarmonizationSuggestions(suggestions=suggestions)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
> Note that the current `SimilaritySearchInMemoryVectorDb` already supports providing a new embedding algorithm.
|
|
99
|
+
|
|
100
|
+
And now, if you have a benchmark and want to evaluate the new approach:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from harmonization.harmonization_benchmark import get_metrics_for_approach
|
|
104
|
+
from harmonization.harmonization_approaches.new_approach import (
|
|
105
|
+
NewApproachExample,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
new_approach = NewApproachExample()
|
|
109
|
+
|
|
110
|
+
output_filename = get_metrics_for_approach(
|
|
111
|
+
benchmark_filepath="path/to/benchmark.jsonl",
|
|
112
|
+
harmonization_approach=new_approach,
|
|
113
|
+
output_filename="output.tsv",
|
|
114
|
+
metrics_column_name="custom_metrics",
|
|
115
|
+
)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Benchmark Details
|
|
119
|
+
|
|
120
|
+
JSONL file, each row is a separate test.
|
|
121
|
+
|
|
122
|
+
Each test should have 3 keys: `input_source_model` in JSON, with desire to harmonize to `input_target_model` in JSON, and we expect known mapping defined in `harmonized_mapping` (which is a TSV represented as a string, with 2 columns `ai_model_node_prop_desc` and `harmonized_model_node_prop_desc`).
|
|
123
|
+
|
|
124
|
+
Example harmonized mapping:
|
|
125
|
+
|
|
126
|
+
| ai_model_node_prop_desc | harmonized_model_node_prop_desc |
|
|
127
|
+
|---|---|
|
|
128
|
+
| PatientData.AgeAtDiagnosis: The age of the patient at the time of diagnosis. | standard_patient_record.age_at_diagnosis: Age in years when diagnosed with the condition. |
|
|
129
|
+
| PatientData.LastVisitDate: The date of the last visit to a healthcare provider. | standard_patient_record.last_visit_date: Date of most recent medical appointment or consultation. |
|
|
130
|
+
| OutpatSkinCheck.AnchorDateOffset: Days between the specified anchor date and the patient's last completed skin test date. | outpat_v_skin_test.DaysFromAnchorDateToEventDate: |
|
|
131
|
+
|
|
132
|
+
The output of the `get_metrics_for_approach` generates a new file with appended metrics per test case in the benchmark.
|
|
133
|
+
|
|
134
|
+
## Local Setup
|
|
135
|
+
|
|
136
|
+
Python 3.12 is recommended, newer versions may also work.
|
|
137
|
+
|
|
138
|
+
This uses [uv](https://docs.astral.sh/uv/) (faster than `pip` and `poetry`, easier to work with).
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
uv sync
|
|
142
|
+
uv pip install -e .
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Tests
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
uv run pytest tests/
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
> Note: Tests are fairly minimal at the moment
|
|
152
|
+
|
|
153
|
+
### Other
|
|
154
|
+
|
|
155
|
+
If for some reason you need to run detect-secrets separately from pre-commit and you have a ton of data or datasets, you can explicitly exclude those directories (they should already ignored by git):
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
detect-secrets scan --exclude-files '.*/data/.*|.*/datasets/.*|.*/output/.*'
|
|
159
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
ai_harmonization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
ai_harmonization/cde.py,sha256=QhCM-b7mYA4NysZd29HANCzSZWf2BO4cuiVIlz2vW34,16469
|
|
3
|
+
ai_harmonization/harmonization_benchmark.py,sha256=_aav8s1ZSHcwI5L1f_He9gygMfG06QL9EjwpWBb_pm4,9693
|
|
4
|
+
ai_harmonization/interactive.py,sha256=vwEGphxFmSySvJcCHWO1u37tJHBbN0KbH1TNxxoXR5A,2291
|
|
5
|
+
ai_harmonization/jsonl.py,sha256=VGIiiyEi_KC06HAoHWxOvDGsaVeIWTIFQgKaxooyjaI,7100
|
|
6
|
+
ai_harmonization/main.py,sha256=OsICCivxDstVNoki1qpNUOW6jjYT_o0x6DsUZHwisAc,336
|
|
7
|
+
ai_harmonization/simple_data_model.py,sha256=cR2IsleppESApTTZeS5ZcRVXK_QWT-mAbtloK8cVc-I,15631
|
|
8
|
+
ai_harmonization/utils.py,sha256=jXWLz55k2cOOBgml7J3ARcEsdSzbEc8B60l-YEdyF_o,18736
|
|
9
|
+
ai_harmonization/harmonization_approaches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
ai_harmonization/harmonization_approaches/base.py,sha256=i3mfQoVLKhA2kyfGZ_gc_ZtbM3H1saodzqnzRgOnP9w,7349
|
|
11
|
+
ai_harmonization/harmonization_approaches/embeddings.py,sha256=HmD4WjYTMOuAfIY0DmjbuYrRd6D1PWu7q6imIfWLe3I,3645
|
|
12
|
+
ai_harmonization/harmonization_approaches/similarity_inmem.py,sha256=zcfzRASDq6RmKrnXfQ14bO_DR4n1EIXsS41I35ByQhI,6188
|
|
13
|
+
ai_harmonization-0.2.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
14
|
+
ai_harmonization-0.2.6.dist-info/METADATA,sha256=OGI3JRQfvfj8RPklusAm2frc0xZUHaCJ-wZVvWLxIAY,5558
|
|
15
|
+
ai_harmonization-0.2.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
16
|
+
ai_harmonization-0.2.6.dist-info/top_level.txt,sha256=FU7_0m8Jr4kcyoFA7Mj-egwa_YxDo-to73WLYeMkESk,17
|
|
17
|
+
ai_harmonization-0.2.6.dist-info/RECORD,,
|