ai_harmonization 0.3.2__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/PKG-INFO +24 -4
  2. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/README.md +18 -2
  3. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/cde.py +26 -20
  4. ai_harmonization-0.5.0/ai_harmonization/dbgap.py +346 -0
  5. ai_harmonization-0.5.0/ai_harmonization/formatters.py +119 -0
  6. ai_harmonization-0.5.0/ai_harmonization/gen3_utils.py +438 -0
  7. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/harmonization_approaches/base.py +8 -0
  8. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/harmonization_approaches/embeddings.py +59 -10
  9. ai_harmonization-0.5.0/ai_harmonization/harmonization_approaches/similarity_inmem.py +474 -0
  10. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/harmonization_benchmark.py +28 -0
  11. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/interactive.py +16 -0
  12. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/jsonl.py +11 -0
  13. ai_harmonization-0.5.0/ai_harmonization/review.py +579 -0
  14. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/simple_data_model.py +299 -9
  15. ai_harmonization-0.5.0/ai_harmonization/styles.py +359 -0
  16. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/utils.py +64 -13
  17. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization.egg-info/PKG-INFO +24 -4
  18. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization.egg-info/SOURCES.txt +13 -1
  19. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization.egg-info/requires.txt +5 -1
  20. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/pyproject.toml +8 -2
  21. ai_harmonization-0.5.0/tests/test_dbgap.py +437 -0
  22. ai_harmonization-0.5.0/tests/test_formatters.py +179 -0
  23. ai_harmonization-0.5.0/tests/test_gen3_utils.py +348 -0
  24. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/tests/test_harmonization.py +4 -3
  25. ai_harmonization-0.5.0/tests/test_review.py +554 -0
  26. ai_harmonization-0.5.0/tests/test_similarity_inmem.py +388 -0
  27. ai_harmonization-0.5.0/tests/test_simple_data_model.py +436 -0
  28. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/tests/test_sssom.py +1 -0
  29. ai_harmonization-0.5.0/tests/test_styles.py +386 -0
  30. ai_harmonization-0.3.2/ai_harmonization/harmonization_approaches/similarity_inmem.py +0 -174
  31. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/LICENSE +0 -0
  32. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/__init__.py +0 -0
  33. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/harmonization_approaches/__init__.py +0 -0
  34. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization/main.py +0 -0
  35. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization.egg-info/dependency_links.txt +0 -0
  36. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/ai_harmonization.egg-info/top_level.txt +0 -0
  37. {ai_harmonization-0.3.2 → ai_harmonization-0.5.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ai_harmonization
3
- Version: 0.3.2
3
+ Version: 0.5.0
4
4
  Summary: AI-backed harmonization framework
5
5
  Requires-Python: <4.0.0,>=3.12
6
6
  Description-Content-Type: text/markdown
@@ -13,8 +13,9 @@ Requires-Dist: pydantic>=2.11.7
13
13
  Requires-Dist: sssom>=0.4.16
14
14
  Requires-Dist: sssom-schema>=1.0.0
15
15
  Requires-Dist: tqdm>=4.67.1
16
- Requires-Dist: transformers>=4.57.1
17
16
  Requires-Dist: itables>=2.4.4
17
+ Requires-Dist: pyyaml>=6.0.1
18
+ Requires-Dist: ipywidgets>=8.1.0
18
19
  Requires-Dist: numpy>=2.0.2
19
20
  Requires-Dist: pandas>=2.2.2
20
21
  Requires-Dist: requests>=2.32.4
@@ -22,6 +23,9 @@ Requires-Dist: ipython>=7.34.0
22
23
  Requires-Dist: torch>=2.8.0
23
24
  Requires-Dist: matplotlib>=3.10.0
24
25
  Requires-Dist: chromadb>=1.0.0
26
+ Requires-Dist: sentence-transformers>=5.1.2
27
+ Requires-Dist: datasets>=4.4.1
28
+ Requires-Dist: accelerate>=1.11.0
25
29
  Dynamic: license-file
26
30
 
27
31
  # AI Harmonization
@@ -36,9 +40,25 @@ pip install ai_harmonization
36
40
 
37
41
  > NOTE: Tested on Python 3.12
38
42
 
39
- ### **AI-Assisted Data Curation Toolkit**
43
+ ### **AI-Assisted Data Harmonization Toolkit**
40
44
 
41
- - **[Demo](./jupyter/ai_assisted_data_curation.ipynb)**
45
+ - **[Demo](https://huggingface.co/spaces/uc-ctds/ai_assisted_data_curation_toolkit/blob/main/ai_assisted_data_curation.ipynb)**
46
+
47
+ ### BioData Catalyst Study Harmonization
48
+
49
+ An end-to-end workflow that maps dbGaP study variables onto the
50
+ [BioData Catalyst Harmonized Model (bdchm)](https://github.com/RTIInternational/NHLBI-BDC-DMC-HM).
51
+ Run the notebooks in order; each one reads what the previous one wrote.
52
+
53
+ 1. [Download Studies Metadata](./jupyter/bdchm/download_studies_data_dictionaries.ipynb) — fetch
54
+ preharmonized PFBs from BDC and extract the dbGaP data dictionary and variable report XMLs.
55
+ 2. [Generate Preliminary Harmonization Mappings](./jupyter/bdchm/harmonize_studies.ipynb) — embed
56
+ the bdchm schema once per prompt variant, then map each study variable to ranked target slots.
57
+ 3. [Review Harmonization Suggestions](./jupyter/bdchm/review_harmonization_suggestions.ipynb) —
58
+ step through the candidates in a widget, accepting or skipping one variable at a time.
59
+
60
+ Steps 1 and 2 write into `inputs/` and `outputs/` relative to the notebook's working directory;
61
+ both are gitignored, as is the `gen3-credentials/` directory holding your Gen3 API key.
42
62
 
43
63
  ### Harmonization Benchmarking and AI Training
44
64
 
@@ -10,9 +10,25 @@ pip install ai_harmonization
10
10
 
11
11
  > NOTE: Tested on Python 3.12
12
12
 
13
- ### **AI-Assisted Data Curation Toolkit**
13
+ ### **AI-Assisted Data Harmonization Toolkit**
14
14
 
15
- - **[Demo](./jupyter/ai_assisted_data_curation.ipynb)**
15
+ - **[Demo](https://huggingface.co/spaces/uc-ctds/ai_assisted_data_curation_toolkit/blob/main/ai_assisted_data_curation.ipynb)**
16
+
17
+ ### BioData Catalyst Study Harmonization
18
+
19
+ An end-to-end workflow that maps dbGaP study variables onto the
20
+ [BioData Catalyst Harmonized Model (bdchm)](https://github.com/RTIInternational/NHLBI-BDC-DMC-HM).
21
+ Run the notebooks in order; each one reads what the previous one wrote.
22
+
23
+ 1. [Download Studies Metadata](./jupyter/bdchm/download_studies_data_dictionaries.ipynb) — fetch
24
+ preharmonized PFBs from BDC and extract the dbGaP data dictionary and variable report XMLs.
25
+ 2. [Generate Preliminary Harmonization Mappings](./jupyter/bdchm/harmonize_studies.ipynb) — embed
26
+ the bdchm schema once per prompt variant, then map each study variable to ranked target slots.
27
+ 3. [Review Harmonization Suggestions](./jupyter/bdchm/review_harmonization_suggestions.ipynb) —
28
+ step through the candidates in a widget, accepting or skipping one variable at a time.
29
+
30
+ Steps 1 and 2 write into `inputs/` and `outputs/` relative to the notebook's working directory;
31
+ both are gitignored, as is the `gen3-credentials/` directory holding your Gen3 API key.
16
32
 
17
33
  ### Harmonization Benchmarking and AI Training
18
34
 
@@ -9,6 +9,29 @@ from langchain_core.documents import Document
9
9
 
10
10
  from ai_harmonization.utils import TEMP_DIR
11
11
 
12
+ # The NIH CDE Repository's one-call export of every data element, as its own
13
+ # search page asks for it.
14
+ SEARCH_EXPORT_URL = "https://cde.nlm.nih.gov/server/de/searchExport"
15
+ SEARCH_EXPORT_QUERY = {
16
+ "resultPerPage": 99999,
17
+ "excludeAllOrgs": False,
18
+ "excludeOrgs": [],
19
+ "includeRetired": False,
20
+ "selectedElements": [],
21
+ "selectedElementsAlt": [],
22
+ "page": 1,
23
+ "includeAggregations": True,
24
+ "selectedAdminStatuses": [],
25
+ "selectedStatuses": [],
26
+ "selectedDatatypes": [],
27
+ "selectedCopyrightStatus": [],
28
+ "searchToken": "",
29
+ "nihEndorsed": False,
30
+ }
31
+ # How long a request waits for the server to connect or to send more data, in
32
+ # seconds. It is not a limit on the whole download.
33
+ TIMEOUT_SECONDS = 180
34
+
12
35
  """
13
36
  Examples:
14
37
 
@@ -77,29 +100,12 @@ def _get_cdes_and_write_to_file() -> str:
77
100
  """
78
101
  Bulk export CDEs from NIH server
79
102
  """
80
- url = "https://cde.nlm.nih.gov/server/de/searchExport"
81
- data = {
82
- "resultPerPage": 99999,
83
- "excludeAllOrgs": False,
84
- "excludeOrgs": [],
85
- "includeRetired": False,
86
- "selectedElements": [],
87
- "selectedElementsAlt": [],
88
- "page": 1,
89
- "includeAggregations": True,
90
- "selectedAdminStatuses": [],
91
- "selectedStatuses": [],
92
- "selectedDatatypes": [],
93
- "selectedCopyrightStatus": [],
94
- "searchToken": "",
95
- "nihEndorsed": False,
96
- }
97
-
98
103
  response = requests.post(
99
- url,
104
+ SEARCH_EXPORT_URL,
100
105
  headers={"Content-Type": "application/json"},
101
- data=json.dumps(data),
106
+ data=json.dumps(SEARCH_EXPORT_QUERY),
102
107
  stream=True,
108
+ timeout=TIMEOUT_SECONDS,
103
109
  )
104
110
 
105
111
  filename = f"{TEMP_DIR}/SearchExport.json"
@@ -0,0 +1,346 @@
1
+ """
2
+ dbGaP data dictionary parsing and mapping output utilities.
3
+
4
+ Parses dbGaP data_dict.xml and var_report.xml files into SimpleDataModel objects
5
+ for use as source variables in harmonization. Also provides helpers for finding
6
+ study files and writing output CSV rows.
7
+ """
8
+
9
+ import logging
10
+ import os
11
+ import re
12
+ import xml.etree.ElementTree as ET
13
+
14
+ import pandas as pd
15
+
16
+ from ai_harmonization.formatters import VALUE_SEPARATOR
17
+ from ai_harmonization.simple_data_model import Node, Property, SimpleDataModel
18
+ from ai_harmonization.styles import STRONG_SIMILARITY, VERY_STRONG_SIMILARITY
19
+
20
+
21
+ DBGAP_TYPE_MAP = {
22
+ "integer": "integer",
23
+ "decimal": "float",
24
+ "float": "float",
25
+ "num": "float",
26
+ "numeric": "float",
27
+ "string": "string",
28
+ "text": "string",
29
+ "encoded value": "string/encoded",
30
+ "encoded": "string/encoded",
31
+ "enum_integer": "string/encoded",
32
+ "date": "datetime",
33
+ "datetime": "datetime",
34
+ }
35
+
36
+ CSV_HEADERS = [
37
+ "Original Node.Property",
38
+ "Suggested Target Node.Property",
39
+ "Similarity",
40
+ "Target Description",
41
+ "Target Values",
42
+ "Original Description",
43
+ "Original Values",
44
+ "study_id",
45
+ "source_table_id",
46
+ "source_variable_name",
47
+ "source_variable_id",
48
+ "prompt_variant",
49
+ "rank",
50
+ ]
51
+
52
+
53
+ def map_dbgap_type(spelling):
54
+ """Map a dbGaP type name to the type that a SimpleDataModel uses.
55
+
56
+ dbGaP can give several type names separated by commas, such as
57
+ ``integer, encoded value``. The first name that DBGAP_TYPE_MAP knows is
58
+ used, so that example is an integer.
59
+
60
+ Args:
61
+ spelling (str | None): The type as dbGaP gives it.
62
+
63
+ Returns:
64
+ str | None: The mapped type, or None if the map knows none of its names.
65
+ """
66
+ for part in (spelling or "").lower().split(","):
67
+ if part.strip() in DBGAP_TYPE_MAP:
68
+ return DBGAP_TYPE_MAP[part.strip()]
69
+ return None
70
+
71
+
72
+ def resolve_variable_type(calculated, declared, has_codes):
73
+ """The type of a dbGaP variable.
74
+
75
+ A variable can have two types: one declared in the study's data
76
+ dictionary, and one calculated by dbGaP from the data itself, in the
77
+ variable report. Declarations are sometimes inaccurate, and calculated
78
+ types are more precise. So a specific calculated type, a number or an
79
+ enumeration, is returned. If the calculated type is the generic ``string``,
80
+ or there is none, the declared type is returned. A variable with value
81
+ codes that is not declared a number is ``string/encoded``, though. If
82
+ neither gives a type, the type is ``string``.
83
+
84
+ Args:
85
+ calculated (str | None): The calculated type, from the variable report.
86
+ declared (str | None): The declared type, from the data dictionary.
87
+ has_codes (bool): Whether the data dictionary lists value codes.
88
+
89
+ Returns:
90
+ str: The variable's type.
91
+ """
92
+ computed = map_dbgap_type(calculated)
93
+ if computed not in (None, "string"):
94
+ return computed
95
+ stated = map_dbgap_type(declared)
96
+ if stated in ("integer", "float"):
97
+ return stated
98
+ if has_codes or stated == "string/encoded":
99
+ return "string/encoded"
100
+ return stated or "string"
101
+
102
+
103
+ def parse_dbgap_table(dict_path, report_path=None):
104
+ """Parse a dbGaP data_dict XML (+ optional var_report XML) into a SimpleDataModel.
105
+
106
+ Each variable's type comes from resolve_variable_type, from its declared
107
+ type in the data dictionary, its calculated type in the report, and
108
+ whether the dictionary lists value codes.
109
+
110
+ Args:
111
+ dict_path (str): Path to the *_data_dict.xml file.
112
+ report_path (str | None): Path to the matching *_var_report.xml file,
113
+ which gives each variable's calculated type; skipped if None or
114
+ missing.
115
+
116
+ Returns:
117
+ tuple[str, SimpleDataModel]: (table_id, model).
118
+ Property.values contains value meanings (for embedding prompts);
119
+ Property.additional_metadata['value_labels'] contains code=meaning
120
+ pairs (for CSV display). Both hold every value the variable
121
+ declares — the prompt formatters cap how many they use, so the
122
+ source and target sides are capped identically. See
123
+ ai_harmonization.formatters.MAX_VALUES_IN_PROMPT.
124
+ Property.additional_metadata['variable_id'] carries the
125
+ variable's dbGaP id, e.g. 'phv99999999.v1'. This id is used for
126
+ variable tracking.
127
+ """
128
+ root = ET.parse(dict_path).getroot()
129
+ table_id = root.attrib.get("id", os.path.basename(dict_path))
130
+
131
+ computed_types = {}
132
+ if report_path and os.path.exists(report_path):
133
+ try:
134
+ for var_node in ET.parse(report_path).getroot().findall(".//variable"):
135
+ v_name = var_node.get("var_name")
136
+ if v_name:
137
+ computed_types[v_name.strip()] = var_node.get("calculated_type")
138
+ except Exception as e:
139
+ # Typed from the dictionary alone rather than half from the report.
140
+ computed_types = {}
141
+ logging.warning(
142
+ f"Could not parse var report {os.path.basename(report_path)}, "
143
+ f"typing its variables from the data dictionary: {e}"
144
+ )
145
+
146
+ properties = []
147
+ for var in root.findall(".//variable"):
148
+ var_name = (var.findtext("name") or "").strip()
149
+ if not var_name:
150
+ continue
151
+
152
+ value_meanings, value_labels = [], []
153
+ for val in var.findall("value"):
154
+ if not val.text:
155
+ continue
156
+ meaning = val.text.strip()
157
+ value_meanings.append(meaning)
158
+ code = val.attrib.get("code")
159
+ value_labels.append(f"{code}={meaning}" if code else meaning)
160
+
161
+ metadata = {}
162
+ if value_labels:
163
+ metadata["value_labels"] = value_labels
164
+ variable_id = var.attrib.get("id")
165
+ if variable_id:
166
+ metadata["variable_id"] = variable_id
167
+
168
+ var_type = resolve_variable_type(
169
+ computed_types.get(var_name), var.findtext("type"), bool(value_meanings)
170
+ )
171
+
172
+ properties.append(
173
+ Property(
174
+ name=var_name,
175
+ description=(
176
+ var.findtext("description") or "No description available."
177
+ ).strip(),
178
+ type=var_type,
179
+ values=value_meanings or None,
180
+ additional_metadata=metadata or None,
181
+ )
182
+ )
183
+
184
+ return table_id, SimpleDataModel(
185
+ nodes=[
186
+ Node(
187
+ name=table_id,
188
+ description="",
189
+ links=[],
190
+ properties=properties,
191
+ )
192
+ ]
193
+ )
194
+
195
+
196
+ def find_study_metadata_files(studies_dir, study_id):
197
+ """Return (study_files_path, dict_files, var_report_by_pht) for a study.
198
+
199
+ Args:
200
+ studies_dir (str): Root directory holding one subdirectory per study.
201
+ study_id (str): e.g. 'phs000557.v7.p2.c1'
202
+
203
+ Returns:
204
+ tuple[str, list[str], dict[str, str]]: The study's metadata directory,
205
+ the *_data_dict.xml filenames in it, and a map from pht accession
206
+ to the full path of the matching *_var_report.xml.
207
+
208
+ Raises:
209
+ FileNotFoundError: If the study's metadata directory does not exist.
210
+ """
211
+ study_files_path = os.path.join(studies_dir, study_id, "metadata")
212
+ if not os.path.exists(study_files_path):
213
+ raise FileNotFoundError(
214
+ f"Path '{study_files_path}' not found. "
215
+ "Run download_studies_data_dictionaries.ipynb first."
216
+ )
217
+
218
+ dict_files = [
219
+ f
220
+ for f in os.listdir(study_files_path)
221
+ if f.lower().endswith(".xml") and "data_dict" in f.lower()
222
+ ]
223
+
224
+ var_report_by_pht = {}
225
+ for f in os.listdir(study_files_path):
226
+ if f.lower().endswith(".xml") and "var_report" in f.lower():
227
+ pht_match = re.search(r"pht\d+", f)
228
+ if pht_match:
229
+ var_report_by_pht[pht_match.group()] = os.path.join(study_files_path, f)
230
+
231
+ return study_files_path, dict_files, var_report_by_pht
232
+
233
+
234
+ def build_mapping_rows(suggestions, slot_values_lookup, study_id):
235
+ """Convert one variable's harmonization suggestions into CSV row dicts.
236
+
237
+ Args:
238
+ suggestions (list[SingleHarmonizationSuggestion]): Suggestions for a
239
+ single source variable, best first — one variable's worth of output
240
+ from MultiPromptSimilaritySearch.
241
+ slot_values_lookup (dict[str, str]): Maps target slot keys to
242
+ enum values joined by VALUE_SEPARATOR.
243
+ study_id (str): Study identifier to embed in every row.
244
+
245
+ Returns:
246
+ list[dict]: One dict per suggestion, keyed by CSV_HEADERS columns.
247
+ ``rank`` is 1-based in descending similarity order.
248
+ """
249
+ rows = []
250
+ for rank, suggestion in enumerate(suggestions, start=1):
251
+ slot_key = f"{suggestion.target_node}.{suggestion.target_property}"
252
+ source_metadata = suggestion.source_additional_metadata or {}
253
+ value_labels = source_metadata.get("value_labels", [])
254
+ rows.append(
255
+ {
256
+ "Original Node.Property": f"{suggestion.source_node}.{suggestion.source_property}",
257
+ "Suggested Target Node.Property": slot_key,
258
+ "Similarity": round(suggestion.similarity, 4),
259
+ "Target Description": suggestion.target_description,
260
+ "Target Values": slot_values_lookup.get(slot_key, ""),
261
+ "Original Description": suggestion.source_description,
262
+ "Original Values": VALUE_SEPARATOR.join(value_labels),
263
+ "study_id": study_id,
264
+ "source_table_id": suggestion.source_node,
265
+ "source_variable_name": suggestion.source_property,
266
+ "source_variable_id": (source_metadata.get("variable_id") or ""),
267
+ "prompt_variant": (suggestion.target_additional_metadata or {}).get(
268
+ "prompt_variant", ""
269
+ ),
270
+ "rank": rank,
271
+ }
272
+ )
273
+ return rows
274
+
275
+
276
+ def generate_variable_mappings(
277
+ study_files_path,
278
+ dict_files,
279
+ var_report_by_pht,
280
+ harmonization_approach,
281
+ slot_values_lookup,
282
+ study_id,
283
+ k,
284
+ ):
285
+ """Generate mapping row dicts for every variable across all tables in a study.
286
+
287
+ Yields one variable at a time so callers can write results incrementally
288
+ rather than holding a whole study's mappings in memory.
289
+
290
+ Args:
291
+ study_files_path (str): Directory containing the XML files.
292
+ dict_files (list[str]): Filenames of *_data_dict.xml files to process.
293
+ var_report_by_pht (dict[str, str]): Maps pht accession to full var_report path.
294
+ harmonization_approach (MultiPromptSimilaritySearch): Search index with
295
+ the target schema already embedded.
296
+ slot_values_lookup (dict[str, str]): Maps slot keys to enum values
297
+ joined by VALUE_SEPARATOR.
298
+ study_id (str): Study identifier embedded in every CSV row.
299
+ k (int): Number of top suggestions to return per variable.
300
+
301
+ Yields:
302
+ list[dict]: CSV row dicts for one source variable (up to k rows).
303
+ """
304
+ for dict_filename in dict_files:
305
+ full_dict_path = os.path.join(study_files_path, dict_filename)
306
+ pht_match = re.search(r"pht\d+", dict_filename)
307
+ full_report_path = (
308
+ var_report_by_pht.get(pht_match.group()) if pht_match else None
309
+ )
310
+
311
+ try:
312
+ _, source_model = parse_dbgap_table(full_dict_path, full_report_path)
313
+ except Exception as e:
314
+ logging.warning(f"Could not parse {dict_filename}, skipping it: {e}")
315
+ continue
316
+
317
+ for suggestions in harmonization_approach.iter_suggestions_by_property(
318
+ source_model, k=k
319
+ ):
320
+ yield build_mapping_rows(suggestions, slot_values_lookup, study_id)
321
+
322
+
323
+ def summarize_rank1_similarity(rank1_group):
324
+ """Compute mapping quality statistics for one study from rank-1 suggestions.
325
+
326
+ Args:
327
+ rank1_group (pd.DataFrame): Rows for a single study_id where rank == 1.
328
+
329
+ Returns:
330
+ pd.Series: Statistics including variable count, mean/median similarity,
331
+ strong-match percentages, and the top bdchm target slot.
332
+ """
333
+ sim = rank1_group["Similarity"]
334
+ strong, very_strong = STRONG_SIMILARITY, VERY_STRONG_SIMILARITY
335
+ return pd.Series(
336
+ {
337
+ "Variables": len(rank1_group),
338
+ "Mean sim (rank 1)": round(sim.mean(), 3),
339
+ "Median sim": round(sim.median(), 3),
340
+ f"≥{strong:.2f} (strong)": f"{(sim >= strong).mean():.0%}",
341
+ f"≥{very_strong:.2f} (very strong)": f"{(sim >= very_strong).mean():.0%}",
342
+ "Top bdchm target": rank1_group.nlargest(1, "Similarity")[
343
+ "Suggested Target Node.Property"
344
+ ].iloc[0],
345
+ }
346
+ )
@@ -0,0 +1,119 @@
1
+ """
2
+ Document formatters for multi-prompt similarity search.
3
+
4
+ Each formatter converts a (Node, Property) pair into a string used for both
5
+ target index documents and source variable queries. Using the same formatter
6
+ on both sides ensures the embedding space is symmetric.
7
+
8
+ The label of the variant that produced a match is recorded in the
9
+ ``prompt_variant`` output column. Pass these to
10
+ ``MultiPromptSimilaritySearch(prompt_variants={'A': ..., 'B': ...})``.
11
+
12
+ Prompt variants:
13
+ A — name + description get_node_property_as_name_description
14
+ B — name + type + description get_node_property_as_name_type_description
15
+ C — name + type + description + enum values get_node_property_as_name_type_description_values
16
+ D — name + type + enum values (no description) get_node_property_as_name_type_values
17
+
18
+ Variants C and D cap how many values they include; see MAX_VALUES_IN_PROMPT.
19
+ """
20
+
21
+ from ai_harmonization.simple_data_model import get_node_property_as_string
22
+
23
+ # What joins a property's values wherever a list of them is rendered as one
24
+ # string: the embedded prompt text, the mapping CSVs, and the review widget.
25
+ #
26
+ # A pipe, not a comma, for two reasons. Some values contain ", " themselves,
27
+ # which makes a comma-joined list impossible to split back into values — and
28
+ # the review widget has only the joined string to work from. And experiments
29
+ # confirmed it matches better.
30
+ VALUE_SEPARATOR = " | "
31
+
32
+ # How many of a property's permissible values go into a prompt.
33
+ #
34
+ # An uncapped list crowds out the property's name, type and description, and is
35
+ # not kept in full anyway: text past the embedding model's input length is
36
+ # discarded wherever the tokenizer reaches it, mid-value. Capping puts that
37
+ # boundary at a whole value, and at the same place on both sides of the
38
+ # comparison, since one formatter builds both the target documents and the
39
+ # source queries. 12 covers all but a handful of real value lists.
40
+ #
41
+ # Only prompt text is affected; the model and the curator's CSV keep every
42
+ # value.
43
+ MAX_VALUES_IN_PROMPT = 12
44
+
45
+
46
+ def format_values(values, limit=MAX_VALUES_IN_PROMPT, separator=VALUE_SEPARATOR):
47
+ """Join a property's permissible values for use in a prompt.
48
+
49
+ Args:
50
+ values (list | None): Permissible values, or None.
51
+ limit (int): Maximum number to include.
52
+ separator (str): What joins the values.
53
+
54
+ Returns:
55
+ str: The values joined by the separator, or "" when there are none.
56
+ """
57
+ if not values:
58
+ return ""
59
+ return separator.join(str(value) for value in values[:limit])
60
+
61
+
62
+ def values_segment(prop):
63
+ """Return the trailing ``" Values: ..."`` segment for a property.
64
+
65
+ The variants that include values all append them the same way — as an
66
+ optional tail with its own leading space — so the segment is built once
67
+ here rather than each variant deciding for itself.
68
+
69
+ Args:
70
+ prop (Property): The property whose values to render.
71
+
72
+ Returns:
73
+ str: ``" Values: a | b | c"``, or ``""`` when the property has none.
74
+ """
75
+ values = format_values(prop.values)
76
+ return f" Values: {values}" if values else ""
77
+
78
+
79
+ def get_node_property_as_name_description(node, prop):
80
+ """Variant A: slot identifier + description (no type)."""
81
+ return f"{node.name}.{prop.name}: {prop.description}"
82
+
83
+
84
+ # Variant B: slot identifier + type + description.
85
+ # Identical to ai_harmonization.simple_data_model.get_node_property_as_string;
86
+ # aliased here so all four prompt-variant formatters live in one place.
87
+ get_node_property_as_name_type_description = get_node_property_as_string
88
+
89
+
90
+ def get_node_property_as_name_type_description_values(node, prop):
91
+ """Variant C: slot identifier + type + description + enum values.
92
+
93
+ Values are capped at MAX_VALUES_IN_PROMPT.
94
+ """
95
+ return (
96
+ f"{node.name}.{prop.name} ({prop.type}): "
97
+ f"{prop.description}{values_segment(prop)}"
98
+ )
99
+
100
+
101
+ def get_node_property_as_name_type_values(node, prop):
102
+ """Variant D: slot identifier + type + enum values — no description.
103
+
104
+ Omitting the description matches on field name, type and value categories
105
+ only. This helps where the two descriptions disagree or where one side is
106
+ boilerplate: a dbGaP variable described as "Severity of emphysema based
107
+ upon the degree of parenchymal involvement" and a target slot described as
108
+ "A subjective assessment of the severity of the condition" share almost no
109
+ wording, while their value ladders line up closely.
110
+
111
+ The trailing colon is load-bearing: ``get_node_prop_type_desc_from_string``
112
+ splits the slot identifier off at the first colon, so without it the parser
113
+ either latches onto the colon in "Values:" and returns a property name of
114
+ "sex (SexEnum) Values", or fails outright and falls back to using the whole
115
+ document as the slot key. Both corrupt the target name silently. The
116
+ ``(type)`` segment, by contrast, is optional as far as that parser is
117
+ concerned — variant A omits it.
118
+ """
119
+ return f"{node.name}.{prop.name} ({prop.type}):{values_segment(prop)}"