ai_harmonization 0.2.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,644 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ from typing import List, Union
5
+
6
+ import requests
7
+ from langchain_community.document_loaders import JSONLoader
8
+ from langchain_core.documents import Document
9
+
10
+ from ai_harmonization.utils import TEMP_DIR
11
+
12
+ """
13
+ Examples:
14
+
15
+ ```json
16
+ {
17
+ "nodes": [
18
+ {
19
+ "name": "read_group_qc_metadata",
20
+ "description": "Metadata related to read groups and quality control metrics.",
21
+ "links": ["foobar"],
22
+ "properties": [
23
+ {
24
+ "description": "The unique identifier for the QC metadata of a read group.",
25
+ "type": "identifier",
26
+ "name": "read_group_qc_id"
27
+ },
28
+ ...
29
+ ]
30
+ },
31
+ ...
32
+ ]
33
+ }
34
+ ```
35
+
36
+ ```json
37
+
38
+ {
39
+ "nodes": [
40
+ {
41
+ "name": "family history",
42
+ "description": "A family history of cancer",
43
+ "links": [
44
+ "subjects"
45
+ ],
46
+ "properties": [
47
+ {
48
+ "description": "The subject's ID",
49
+ "type": "string",
50
+ "propertyName": "subjects_id"
51
+ },
52
+ ...
53
+ ]
54
+ },
55
+ ...
56
+ ```
57
+
58
+ """
59
+
60
+
61
+ def get_cdes_as_langchain_documents() -> List[Document]:
62
+ """
63
+ Retrieves and converts CDEs to LangChain documents.
64
+
65
+ Returns:
66
+ List[Document]: A list of Document objects representing the converted LangChain documents.
67
+ """
68
+ os.makedirs(TEMP_DIR, exist_ok=True)
69
+
70
+ input_file_path = _get_cdes_and_write_to_file()
71
+ filtered_file_path = _convert_cdes_to_jsonl_and_filter(input_file_path)
72
+ documents = _convert_filtered_cdes_to_langchain_documents(filtered_file_path)
73
+ return documents
74
+
75
+
76
+ def _get_cdes_and_write_to_file() -> str:
77
+ """
78
+ Bulk export CDEs from NIH server
79
+ """
80
+ url = "https://cde.nlm.nih.gov/server/de/searchExport"
81
+ data = {
82
+ "resultPerPage": 99999,
83
+ "excludeAllOrgs": False,
84
+ "excludeOrgs": [],
85
+ "includeRetired": False,
86
+ "selectedElements": [],
87
+ "selectedElementsAlt": [],
88
+ "page": 1,
89
+ "includeAggregations": True,
90
+ "selectedAdminStatuses": [],
91
+ "selectedStatuses": [],
92
+ "selectedDatatypes": [],
93
+ "selectedCopyrightStatus": [],
94
+ "searchToken": "",
95
+ "nihEndorsed": False,
96
+ }
97
+
98
+ response = requests.post(
99
+ url,
100
+ headers={"Content-Type": "application/json"},
101
+ data=json.dumps(data),
102
+ stream=True,
103
+ )
104
+
105
+ filename = f"{TEMP_DIR}/SearchExport.json"
106
+ with open(filename, "wb") as file:
107
+ for chunk in response.iter_content(1024):
108
+ file.write(chunk)
109
+
110
+ logging.info(f"Wrote raw CDEs to {filename}")
111
+ return filename
112
+
113
+
114
+ def _convert_cdes_to_jsonl_and_filter(input_file_path: str) -> str:
115
+ """
116
+ Converts a JSONL file to a filtered JSONL file by selecting only
117
+ relevant metadata for the CDEs.
118
+
119
+ Args:
120
+ input_file_path: The path to the input JSONL file.
121
+
122
+ Returns:
123
+ str: The path to the filtered JSONL file.
124
+ """
125
+ output_file_path = os.path.splitext(input_file_path)[0] + ".jsonl"
126
+ filtered_file_path = os.path.splitext(input_file_path)[0] + "_filtered.jsonl"
127
+
128
+ with (
129
+ open(input_file_path, "r") as input_file,
130
+ open(output_file_path, "w") as output_file,
131
+ ):
132
+ raw_input = json.load(input_file)
133
+ for item in raw_input:
134
+ output_file.write(json.dumps(item) + "\n")
135
+
136
+ # This extracts and filters metadata from the record
137
+ with (
138
+ open(output_file_path) as jsonl_file,
139
+ open(filtered_file_path, "w") as filtered_output,
140
+ ):
141
+ for row in jsonl_file:
142
+ item = json.loads(row)
143
+ info = {
144
+ "tinyID": item["tinyId"],
145
+ "names": [item["designation"] for item in item["designations"]],
146
+ "dataElementConcept": item["dataElementConcept"],
147
+ "steward": item["steward"],
148
+ "definitions": [item["definition"] for item in item["definitions"]],
149
+ "properties": [item["value"] for item in item["properties"]],
150
+ "elementType": item["elementType"],
151
+ "ids": [item["id"] for item in item["ids"]],
152
+ "nihEndorsed": item["nihEndorsed"],
153
+ "valueDomain": item["valueDomain"],
154
+ }
155
+ filtered_output.write(json.dumps(info) + "\n")
156
+
157
+ logging.info(f"Wrote filtered CDEs to {filtered_file_path}")
158
+ return filtered_file_path
159
+
160
+
161
+ def _convert_filtered_cdes_to_langchain_documents(
162
+ filtered_file_path: str, metadata_func: Union[callable, None] = None
163
+ ) -> List[Document]:
164
+ """
165
+ Converts a filtered CDEs JSON file to a list of Langchain documents.
166
+
167
+ Args:
168
+ filtered_file_path: The path to the filtered CDEs JSON file.
169
+
170
+ Returns:
171
+ List[Document]: A list of Langchain Document objects.
172
+ """
173
+ metadata_func = metadata_func or _metadata_func
174
+ loader = JSONLoader(
175
+ file_path=filtered_file_path,
176
+ jq_schema=".",
177
+ text_content=False,
178
+ json_lines=True,
179
+ content_key="names",
180
+ metadata_func=metadata_func,
181
+ )
182
+
183
+ documents = loader.load()
184
+
185
+ return documents
186
+
187
+
188
+ def _metadata_func(record: dict, metadata: dict) -> dict:
189
+ """This converts non-strings to string so all this can be tokenized"""
190
+ metadata["names"] = str(record.get("names", []))
191
+ metadata["dataElementConcept"] = str(record.get("dataElementConcept"))
192
+ metadata["steward"] = str(record.get("steward"))
193
+ metadata["definitions"] = str(record.get("definitions", []))
194
+ metadata["properties"] = str(record.get("properties", []))
195
+ metadata["ids"] = str(record.get("ids", []))
196
+ metadata["nihEndorsed"] = record.get("nihEndorsed", False)
197
+ metadata["valueDomain"] = str(record.get("valueDomain", []))
198
+
199
+ metadata["source"] = f"https://cde.nlm.nih.gov/deView?tinyId={record['tinyID']}"
200
+
201
+ return metadata
202
+
203
+
204
+ EXAMPLE_AI_MODEL_OUTPUT_JSON = json.loads(
205
+ """
206
+ {
207
+ "nodes": [
208
+ {
209
+ "name": "submitted_aligned_reads_file_manifest",
210
+ "description": "Manifest of submitted aligned reads files",
211
+ "links": [
212
+ "projects_id"
213
+ ]
214
+ ,
215
+ "properties": [
216
+ {
217
+ "description": "Unique identifier for the submitted aligned reads file",
218
+ "type": "integer",
219
+ "name": "submitted_aligned_reads_id"
220
+ }
221
+ ,
222
+ {
223
+ "description": "Name of the submitted aligned reads file",
224
+ "type": "string",
225
+ "name": "file_name"
226
+ }
227
+ ,
228
+ {
229
+ "description": "Format of the submitted aligned reads file",
230
+ "type": "string",
231
+ "name": "data_format"
232
+ }
233
+ ,
234
+ {
235
+ "description": "Category of the submitted aligned reads file",
236
+ "type": "string",
237
+ "name": "data_category"
238
+ }
239
+ ,
240
+ {
241
+ "description": "Type of the submitted aligned reads file",
242
+ "type": "string",
243
+ "name": "data_type"
244
+ }
245
+ ,
246
+ {
247
+ "description": "Experimental strategy used for the submitted aligned reads file",
248
+ "type": "string",
249
+ "name": "experimental_strategy"
250
+ }
251
+ ,
252
+ {
253
+ "description": "Consent codes associated with the submitted aligned reads file",
254
+ "type": "string",
255
+ "name": "consent_codes"
256
+ }
257
+ ,
258
+ {
259
+ "description": "Projects ID associated with the submitted aligned reads file",
260
+ "type": "integer",
261
+ "name": "projects_id"
262
+ }
263
+ ,
264
+ {
265
+ "description": "Cases ID associated with the submitted aligned reads file",
266
+ "type": "integer",
267
+ "name": "cases_id"
268
+ }
269
+ ,
270
+ {
271
+ "description": "Samples ID associated with the submitted aligned reads file",
272
+ "type": "integer",
273
+ "name": "samples_id"
274
+ }
275
+ ,
276
+ {
277
+ "description": "Aliquots ID associated with the submitted aligned reads file",
278
+ "type": "integer",
279
+ "name": "aliquots_id"
280
+ }
281
+ ,
282
+ {
283
+ "description": "Read groups ID associated with the submitted aligned reads file",
284
+ "type": "integer",
285
+ "name": "read_groups_id"
286
+ }
287
+ ]
288
+ }
289
+ ,
290
+ {
291
+ "name": "aligned_reads_index_file_manifest",
292
+ "description": "Manifest of aligned reads index files",
293
+ "links": [
294
+ "submitted_aligned_reads_files_id"
295
+ ]
296
+ ,
297
+ "properties": [
298
+ {
299
+ "description": "Unique identifier for the aligned reads index file",
300
+ "type": "integer",
301
+ "name": "aligned_reads_index_id"
302
+ }
303
+ ,
304
+ {
305
+ "description": "Category of the aligned reads index file",
306
+ "type": "string",
307
+ "name": "data_category"
308
+ }
309
+ ,
310
+ {
311
+ "description": "Type of the aligned reads index file",
312
+ "type": "string",
313
+ "name": "data_type"
314
+ }
315
+ ,
316
+ {
317
+ "description": "Format of the aligned reads index file",
318
+ "type": "string",
319
+ "name": "data_format"
320
+ }
321
+ ,
322
+ {
323
+ "description": "ID of the submitted aligned reads file associated with this index file",
324
+ "type": "integer",
325
+ "name": "submitted_aligned_reads_files_id"
326
+ }
327
+ ]
328
+ }
329
+ ,
330
+ {
331
+ "name": "read_group_metadata",
332
+ "description": "Metadata of read groups",
333
+ "links": [
334
+ "projects_id",
335
+ "cases_id",
336
+ "samples_id",
337
+ "aliquots_id"
338
+ ]
339
+ ,
340
+ "properties": [
341
+ {
342
+ "description": "ID of the read group",
343
+ "type": "integer",
344
+ "name": "read_group_id"
345
+ }
346
+ ,
347
+ {
348
+ "description": "Age at diagnosis",
349
+ "type": "integer",
350
+ "name": "age_at_diagnosis"
351
+ }
352
+ ,
353
+ {
354
+ "description": "Age at index",
355
+ "type": "integer",
356
+ "name": "age_at_index"
357
+ }
358
+ ,
359
+ {
360
+ "description": "Aliquot description",
361
+ "type": "string",
362
+ "name": "aliquot_description"
363
+ }
364
+ ,
365
+ {
366
+ "description": "Amount of the sample",
367
+ "type": "float",
368
+ "name": "amount"
369
+ }
370
+ ,
371
+ {
372
+ "description": "Date of analyte isolation",
373
+ "type": "date",
374
+ "name": "analyte_isolation_date"
375
+ }
376
+ ,
377
+ {
378
+ "description": "Quantity of the analyte",
379
+ "type": "float",
380
+ "name": "analyte_quantity"
381
+ }
382
+ ,
383
+ {
384
+ "description": "Type of the analyte",
385
+ "type": "string",
386
+ "name": "analyte_type_id"
387
+ }
388
+ ,
389
+ {
390
+ "description": "Physical site of biospecimen",
391
+ "type": "string",
392
+ "name": "biospecimen_physical_site"
393
+ }
394
+ ,
395
+ {
396
+ "description": "Birth year",
397
+ "type": "integer",
398
+ "name": "birth_year"
399
+ }
400
+ ,
401
+ {
402
+ "description": "Other composition of the sample",
403
+ "type": "string",
404
+ "name": "composition_other"
405
+ }
406
+ ,
407
+ {
408
+ "description": "Method of contrivance",
409
+ "type": "string",
410
+ "name": "contrivance_method"
411
+ }
412
+ ,
413
+ {
414
+ "description": "County (long)",
415
+ "type": "string",
416
+ "name": "county_long"
417
+ }
418
+ ,
419
+ {
420
+ "description": "Cycles",
421
+ "type": "integer",
422
+ "name": "cycles"
423
+ }
424
+ ,
425
+ {
426
+ "description": "Date of collection",
427
+ "type": "date",
428
+ "name": "date_collected"
429
+ }
430
+ ,
431
+ {
432
+ "description": "Days to assay",
433
+ "type": "integer",
434
+ "name": "days_to_assay"
435
+ }
436
+ ,
437
+ {
438
+ "description": "Days to lost to followup",
439
+ "type": "integer",
440
+ "name": "days_to_lost_to_followup"
441
+ }
442
+ ,
443
+ {
444
+ "description": "Mean fragment length",
445
+ "type": "float",
446
+ "name": "fragment_mean_length"
447
+ }
448
+ ,
449
+ {
450
+ "description": "Minimum fragment length",
451
+ "type": "float",
452
+ "name": "fragment_minimum_length"
453
+ }
454
+ ,
455
+ {
456
+ "description": "ID of the GDC sample",
457
+ "type": "string",
458
+ "name": "gdc_sample_id"
459
+ }
460
+ ,
461
+ {
462
+ "description": "GenBank accession",
463
+ "type": "string",
464
+ "name": "genbank_accession"
465
+ }
466
+ ,
467
+ {
468
+ "description": "Host name or IP address of the submission tool",
469
+ "type": "string",
470
+ "name": "host_name"
471
+ }
472
+ ,
473
+ {
474
+ "description": "ID of the host",
475
+ "type": "integer",
476
+ "name": "id"
477
+ }
478
+ ,
479
+ {
480
+ "description": "Institution name",
481
+ "type": "string",
482
+ "name": "institution"
483
+ }
484
+ ,
485
+ {
486
+ "description": "International classification of diseases code",
487
+ "type": "string",
488
+ "name": "icd_code"
489
+ }
490
+ ,
491
+ {
492
+ "description": "ID of the project",
493
+ "type": "integer",
494
+ "name": "id"
495
+ }
496
+ ,
497
+ {
498
+ "description": "ID of the primary accession",
499
+ "type": "string",
500
+ "name": "id"
501
+ }
502
+ ,
503
+ {
504
+ "description": "Index (0-based)",
505
+ "type": "integer",
506
+ "name": "index"
507
+ }
508
+ ,
509
+ {
510
+ "description": "Illumina platform",
511
+ "type": "string",
512
+ "name": "illumina_platform"
513
+ }
514
+ ,
515
+ {
516
+ "description": "ID of the library",
517
+ "type": "integer",
518
+ "name": "id"
519
+ }
520
+ ,
521
+ {
522
+ "description": "Library strategy",
523
+ "type": "string",
524
+ "name": "library_strategy"
525
+ }
526
+ ,
527
+ {
528
+ "description": "Library type",
529
+ "type": "string",
530
+ "name": "library_type"
531
+ }
532
+ ,
533
+ {
534
+ "description": "Method of library preparation",
535
+ "type": "string",
536
+ "name": "method_library_preparation"
537
+ }
538
+ ,
539
+ {
540
+ "description": "Name of the file",
541
+ "type": "string",
542
+ "name": "name"
543
+ }
544
+ ,
545
+ {
546
+ "description": "ID of the patient",
547
+ "type": "integer",
548
+ "name": "id"
549
+ }
550
+ ,
551
+ {
552
+ "description": "Platform (e.g., Illumina)",
553
+ "type": "string",
554
+ "name": "platform"
555
+ }
556
+ ,
557
+ {
558
+ "description": "ID of the project",
559
+ "type": "integer",
560
+ "name": "project_id"
561
+ }
562
+ ,
563
+ {
564
+ "description": "Protocol name (e.g., Illumina TruSeq)",
565
+ "type": "string",
566
+ "name": "protocol_name"
567
+ }
568
+ ,
569
+ {
570
+ "description": "ID of the read group",
571
+ "type": "integer",
572
+ "name": "id"
573
+ }
574
+ ,
575
+ {
576
+ "description": "Read group status",
577
+ "type": "string",
578
+ "name": "read_group_status"
579
+ }
580
+ ,
581
+ {
582
+ "description": "ID of the sample",
583
+ "type": "integer",
584
+ "name": "sample_id"
585
+ }
586
+ ,
587
+ {
588
+ "description": "Scientific name of the organism",
589
+ "type": "string",
590
+ "name": "scientific_name"
591
+ }
592
+ ,
593
+ {
594
+ "description": "ID of the submission tool",
595
+ "type": "integer",
596
+ "name": "submission_tool_id"
597
+ }
598
+ ,
599
+ {
600
+ "description": "Submission timestamp (in seconds)",
601
+ "type": "integer",
602
+ "name": "submission_timestamp"
603
+ }
604
+ ,
605
+ {
606
+ "description": "ID of the study",
607
+ "type": "integer",
608
+ "name": "study_id"
609
+ }
610
+ ,
611
+ {
612
+ "description": "Study name",
613
+ "type": "string",
614
+ "name": "study_name"
615
+ }
616
+ ,
617
+ {
618
+ "description": "Submission tool (e.g., Illumina BCLConvert)",
619
+ "type": "string",
620
+ "name": "submission_tool"
621
+ }
622
+ ,
623
+ {
624
+ "description": "Submission timestamp (in seconds)",
625
+ "type": "integer",
626
+ "name": "submission_timestamp"
627
+ }
628
+ ,
629
+ {
630
+ "description": "Type of the sample",
631
+ "type": "string",
632
+ "name": "sample_type"
633
+ }
634
+ ,
635
+ {
636
+ "description": "ID of the submission tool",
637
+ "type": "integer",
638
+ "name": "id"
639
+ }
640
+ ]
641
+ }
642
+ ]}
643
+ """
644
+ )
File without changes