ai_harmonization 0.2.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,202 @@
1
+ import os
2
+ from abc import ABC, abstractmethod
3
+ from typing import Dict, List, Optional
4
+
5
+ import pandas as pd
6
+ from pydantic import BaseModel
7
+ from sssom.parsers import MappingSetDataFrame
8
+ from sssom.writers import write_table
9
+ from sssom_schema import Mapping, MappingSet
10
+
11
+ from ai_harmonization.simple_data_model import SimpleDataModel
12
+ from ai_harmonization.utils import nodeproperty_to_curie
13
+
14
+ DEFAULT_SSSOM_METADATA: Dict[str, str | dict] = {
15
+ "mapping_set_id": "https://example.org/mappings/example.sssom.tsv",
16
+ "license": "https://www.apache.org/licenses/LICENSE-2.0",
17
+ "mapping_set_description": (
18
+ "This mapping set aligns properties (columns/fields) between source and target data models (graph or relational). "
19
+ "For each mapping, subject_id and object_id are composite identifiers in the form 'node.property', "
20
+ "where 'node' is the original entity/table/class, and 'property' is the attribute/field/column. "
21
+ "For example, 'Subject.name' refers to the 'name' property of the 'Subject' node in the source model."
22
+ ),
23
+ }
24
+ SSSOM_PREFIX_MAP = {"EX": "http://example.org/"}
25
+ DEFAULT_SSSOM_PREDICATE_ID: str = "skos:relatedMatch"
26
+ DEFAULT_SSSOM_MAPPING_JUSTIFICATION: str = "semapv:UnspecifiedMatching"
27
+
28
+
29
+ class SingleHarmonizationSuggestion(BaseModel):
30
+ """
31
+ Represents a harmonization suggestion for a pair of node.property attributes
32
+ """
33
+
34
+ source_node: str
35
+ source_property: str
36
+ source_description: str
37
+ source_additional_metadata: Optional[dict] = None
38
+ target_node: str
39
+ target_property: str
40
+ target_description: str
41
+ target_additional_metadata: Optional[dict] = None
42
+ similarity: Optional[float] = None
43
+
44
+
45
+ class HarmonizationSuggestions(BaseModel):
46
+ """
47
+ Container for a batch of harmonization suggestions.
48
+ """
49
+
50
+ suggestions: List[SingleHarmonizationSuggestion]
51
+
52
+ def to_dataframe(self) -> pd.DataFrame:
53
+ return pd.DataFrame([suggestion.__dict__ for suggestion in self.suggestions])
54
+
55
+ def to_simlified_dataframe(self) -> pd.DataFrame:
56
+ data = []
57
+ for suggestion in self.suggestions:
58
+ data.append(
59
+ {
60
+ "Original Node.Property": f"{suggestion.source_node}.{suggestion.source_property}",
61
+ "Suggested Target Node.Property": f"{suggestion.target_node}.{suggestion.target_property}",
62
+ "Similarity": suggestion.similarity,
63
+ "Target Description": suggestion.target_description,
64
+ "Original Description": suggestion.source_description,
65
+ }
66
+ )
67
+ return pd.DataFrame(data)
68
+
69
+
70
+ class HarmonizationApproach(ABC):
71
+ @abstractmethod
72
+ def get_harmonization_suggestions(
73
+ self,
74
+ input_source_model: SimpleDataModel,
75
+ input_target_model: SimpleDataModel,
76
+ **kwargs,
77
+ ) -> HarmonizationSuggestions:
78
+ """
79
+ Returns HarmonizationSuggestions according to the implementing algorithm.
80
+ """
81
+ raise NotImplementedError()
82
+
83
+
84
+ class ExampleHarmonizationApproach(HarmonizationApproach):
85
+ """
86
+ Dummy example that returns a static harmonization suggestion.
87
+ """
88
+
89
+ def get_harmonization_suggestions(
90
+ self,
91
+ input_source_model: SimpleDataModel,
92
+ input_target_model: SimpleDataModel,
93
+ **kwargs,
94
+ ) -> HarmonizationSuggestions:
95
+ """
96
+ Example implementation for demonstration.
97
+ """
98
+
99
+ suggestions: List[SingleHarmonizationSuggestion] = [
100
+ SingleHarmonizationSuggestion(
101
+ source_node="source_node_1",
102
+ source_property="source_property_1",
103
+ source_description="source_description_1",
104
+ source_additional_metadata={"type": "string"},
105
+ target_node="target_node_1",
106
+ target_property="target_property_1",
107
+ target_description="target_description_1",
108
+ target_additional_metadata={"type": "number"},
109
+ similarity=0.9,
110
+ )
111
+ ]
112
+ return HarmonizationSuggestions(suggestions=suggestions)
113
+
114
+
115
+ def harmonization_suggestions_to_sssom(
116
+ suggestions: List[SingleHarmonizationSuggestion],
117
+ filename: str,
118
+ meta: Optional[dict] = None,
119
+ exclude_required_comments: bool = False,
120
+ ) -> None:
121
+ """
122
+ Convert a list of harmonization suggestions into a MappingSet and write to a SSSOM TSV file.
123
+
124
+ Args:
125
+ suggestions: List of SingleHarmonizationSuggestion.
126
+ filename: Output .sssom.tsv file.
127
+ meta: Optional SSSOM MappingSet metadata dictionary. If None, uses DEFAULT_SSSOM_METADATA.
128
+ exclude_required_comments: Will output a non-standard SSSOM file with no comments. This allows
129
+ you to create a "clean" TSV which starts with headers and contains only rows after
130
+ """
131
+ meta = meta or DEFAULT_SSSOM_METADATA
132
+ mappings = [
133
+ suggestion_to_mapping(single_suggestion) for single_suggestion in suggestions
134
+ ]
135
+ mapping_set = MappingSet(**meta, mappings=mappings)
136
+ write_mappingset_to_sssom(mapping_set, filename)
137
+
138
+ if exclude_required_comments:
139
+ with open(filename, "r") as file:
140
+ lines = [line for line in file if not line.lstrip().startswith("#")]
141
+ with open(filename, "w") as file:
142
+ file.writelines(lines)
143
+
144
+
145
+ def suggestion_to_mapping(
146
+ suggestion: SingleHarmonizationSuggestion,
147
+ predicate_id: Optional[str] = None,
148
+ mapping_justification: Optional[str] = None,
149
+ ) -> Mapping:
150
+ """
151
+ Convert a SingleHarmonizationSuggestion to an SSSOM Mapping.
152
+
153
+ Args:
154
+ suggestion: The harmonization suggestion to convert.
155
+ predicate_id: Which predicate to use; default is DEFAULT_SSSOM_PREDICATE_ID.
156
+ mapping_justification: Which mapping justification to use; default is DEFAULT_SSSOM_MAPPING_JUSTIFICATION.
157
+
158
+ Returns:
159
+ Mapping: The equivalent SSSOM Mapping object.
160
+ """
161
+ predicate_id = predicate_id or DEFAULT_SSSOM_PREDICATE_ID
162
+ mapping_justification = mapping_justification or DEFAULT_SSSOM_MAPPING_JUSTIFICATION
163
+
164
+ # SSSOM requires CURIEs or URLs.
165
+ subject_id = nodeproperty_to_curie(
166
+ f"{suggestion.source_node}.{suggestion.source_property}"
167
+ )
168
+ object_id = nodeproperty_to_curie(
169
+ f"{suggestion.target_node}.{suggestion.target_property}"
170
+ )
171
+
172
+ return Mapping(
173
+ predicate_id=predicate_id,
174
+ mapping_justification=mapping_justification,
175
+ subject_id=subject_id,
176
+ subject_label=suggestion.source_description,
177
+ object_id=object_id,
178
+ object_label=suggestion.target_description,
179
+ object_source=None, # TODO: data dictionary. note there's also a version
180
+ similarity_score=suggestion.similarity,
181
+ )
182
+
183
+
184
+ def write_mappingset_to_sssom(
185
+ mapping_set: MappingSet,
186
+ filename: str,
187
+ ) -> None:
188
+ """
189
+ Write a MappingSet object to a SSSOM TSV file.
190
+
191
+ Args:
192
+ mapping_set: The SSSOM MappingSet to write.
193
+ filename: The path to write the .sssom.tsv file.
194
+ """
195
+ os.makedirs(os.path.dirname(filename), exist_ok=True)
196
+
197
+ mapping_set_df = MappingSetDataFrame.from_mapping_set(
198
+ mapping_set, converter=SSSOM_PREFIX_MAP
199
+ )
200
+ mapping_set_df.clean_prefix_map()
201
+ with open(filename, "w", encoding="utf-8") as output_file:
202
+ write_table(mapping_set_df, output_file)
@@ -0,0 +1,97 @@
1
+ import numpy as np
2
+ import torch
3
+ from transformers import AutoModel, AutoTokenizer
4
+
5
+
6
+ class BaseEmbeddings:
7
+ def __init__(self, model_name, device=None, trust_remote_code=False):
8
+ # Load tokenizer/model, set device, set eval()
9
+ self.tokenizer = AutoTokenizer.from_pretrained(
10
+ model_name, trust_remote_code=trust_remote_code
11
+ )
12
+ self.device = torch.device(
13
+ device if device else ("cuda" if torch.cuda.is_available() else "cpu")
14
+ )
15
+ self.model = (
16
+ AutoModel.from_pretrained(model_name, trust_remote_code=trust_remote_code)
17
+ .to(self.device)
18
+ .eval()
19
+ )
20
+
21
+ def embed_query(self, text):
22
+ return self.embed_documents([text])[0]
23
+
24
+ def mean_pool(self, last_hidden, mask):
25
+ # Shape: (batch, seq_len, dim), mask: (batch, seq_len, 1)
26
+ masked = last_hidden * mask
27
+ summed = masked.sum(1)
28
+ counts = mask.sum(1)
29
+ mean_pooled = summed / counts.clamp(min=1e-9)
30
+ return mean_pooled
31
+
32
+ def embed_documents(self, texts):
33
+ raise NotImplementedError("Implement in subclass.")
34
+
35
+
36
+ class MedGemmaEmbeddings(BaseEmbeddings):
37
+ def __init__(self, model_name="google/medgemma-4b-it", device=None):
38
+ super().__init__(model_name, device, trust_remote_code=True)
39
+
40
+ def embed_documents(self, texts):
41
+ inputs = self.tokenizer(
42
+ texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
43
+ ).to(self.device)
44
+ with torch.no_grad():
45
+ outputs = self.model(**inputs, output_hidden_states=True)
46
+ last_hidden = outputs.hidden_states[-1]
47
+ mask = (
48
+ inputs["attention_mask"]
49
+ .unsqueeze(-1)
50
+ .expand(last_hidden.size())
51
+ .float()
52
+ )
53
+ pooled = self.mean_pool(last_hidden, mask)
54
+ return pooled.cpu().numpy().tolist()
55
+
56
+
57
+ class QwenEmbeddings(BaseEmbeddings):
58
+ def __init__(self, model_name="Qwen/Qwen3-Embedding-0.6B", device=None):
59
+ super().__init__(model_name, device, trust_remote_code=True)
60
+
61
+ def embed_documents(self, texts):
62
+ inputs = self.tokenizer(
63
+ texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
64
+ ).to(self.device)
65
+ with torch.no_grad():
66
+ outputs = self.model(**inputs, output_hidden_states=True)
67
+ # recommended by Qwen
68
+ pooled = outputs.get("pooled_output", None)
69
+ if pooled is not None:
70
+ return pooled.cpu().numpy().tolist()
71
+ # fallback to mean pool
72
+ last_hidden = outputs.hidden_states[-1]
73
+ mask = (
74
+ inputs["attention_mask"]
75
+ .unsqueeze(-1)
76
+ .expand(last_hidden.size())
77
+ .float()
78
+ )
79
+ pooled = self.mean_pool(last_hidden, mask)
80
+ return pooled.cpu().numpy().tolist()
81
+
82
+
83
+ class BGEEmbeddings(BaseEmbeddings):
84
+ def __init__(self, model_name="BAAI/bge-large-en-v1.5", device=None):
85
+ super().__init__(model_name, device=device)
86
+
87
+ def embed_documents(self, texts):
88
+ inputs = self.tokenizer(
89
+ texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
90
+ ).to(self.device)
91
+ with torch.no_grad():
92
+ model_output = self.model(**inputs)
93
+ mask = inputs["attention_mask"].unsqueeze(-1)
94
+ pooled = self.mean_pool(model_output.last_hidden_state, mask)
95
+ pooled = pooled.cpu().numpy()
96
+ pooled = pooled / np.linalg.norm(pooled, axis=1, keepdims=True)
97
+ return pooled.tolist()
@@ -0,0 +1,174 @@
1
+ import logging
2
+ from typing import List, TypedDict
3
+
4
+ from langchain_core.documents import Document
5
+ from langchain_huggingface import HuggingFaceEmbeddings
6
+
7
+ from ai_harmonization.harmonization_approaches.base import (
8
+ HarmonizationApproach,
9
+ HarmonizationSuggestions,
10
+ SingleHarmonizationSuggestion,
11
+ )
12
+ from ai_harmonization.simple_data_model import (
13
+ Node,
14
+ Property,
15
+ SimpleDataModel,
16
+ get_data_model_as_langchain_documents,
17
+ get_node_prop_type_desc_from_string,
18
+ get_node_property_as_string,
19
+ )
20
+ from ai_harmonization.utils import (
21
+ TEMP_DIR,
22
+ add_documents_to_vectorstore,
23
+ get_langchain_vectorstore_and_persistent_client,
24
+ get_similar_documents,
25
+ )
26
+
27
+
28
+ class ExistingVectorstoreException(BaseException):
29
+ pass
30
+
31
+
32
+ class SuggestionInfo(TypedDict):
33
+ node: Node
34
+ property: Property
35
+ matches: List[Document]
36
+
37
+
38
+ class SimilaritySearchInMemoryVectorDb(HarmonizationApproach):
39
+
40
+ def __init__(
41
+ self,
42
+ vectordb_persist_directory_name: str,
43
+ input_target_model: SimpleDataModel,
44
+ embedding_function: HuggingFaceEmbeddings | None = None,
45
+ force_vectorstore_recreation: bool = False,
46
+ batch_size: int | None = None,
47
+ ):
48
+ super().__init__()
49
+
50
+ self.embedding_function = embedding_function or HuggingFaceEmbeddings(
51
+ model_name="sentence-transformers/all-mpnet-base-v2"
52
+ )
53
+ self.vectordb_persist_directory_name = vectordb_persist_directory_name
54
+ vectorstore, persistent_client = (
55
+ get_langchain_vectorstore_and_persistent_client(
56
+ persist_directory_name=self.vectordb_persist_directory_name,
57
+ embedding_function=self.embedding_function,
58
+ )
59
+ )
60
+ self.vectorstore = vectorstore
61
+ self.persistent_client = persistent_client
62
+
63
+ try:
64
+ self._add_target_to_vector_database(
65
+ input_target_model,
66
+ force_recreation=force_vectorstore_recreation,
67
+ batch_size=batch_size,
68
+ )
69
+ except ExistingVectorstoreException:
70
+ logging.info("vectorstore already exists, NOT recreating...")
71
+ pass
72
+
73
+ def _add_target_to_vector_database(
74
+ self, input_target_model, force_recreation=False, batch_size=None
75
+ ):
76
+ if force_recreation:
77
+ all_ids = self.vectorstore.get()["ids"]
78
+ if all_ids:
79
+ self.vectorstore.delete(ids=all_ids)
80
+
81
+ if len(self.vectorstore.get()["ids"]) != 0:
82
+ raise ExistingVectorstoreException(
83
+ f"Vectorstore in persist directory {self.vectordb_persist_directory_name} already has data here: {TEMP_DIR}, "
84
+ "aborting re-adding. Delete the persist directory, use a new one, or force recreation."
85
+ )
86
+
87
+ target_docs = get_data_model_as_langchain_documents(input_target_model)
88
+
89
+ add_documents_to_vectorstore(
90
+ target_docs, self.vectorstore, self.persistent_client, batch_size
91
+ )
92
+
93
+ def get_harmonization_suggestions(
94
+ self,
95
+ input_source_model: SimpleDataModel,
96
+ input_target_model: SimpleDataModel,
97
+ **kwargs,
98
+ ):
99
+ # note: k and score_threshold are in kwargs
100
+ suggestions_for_output_model = self._get_suggestions_for_source_model(
101
+ input_source_model, **kwargs
102
+ )
103
+ suggestions = []
104
+ for suggestion_info in suggestions_for_output_model:
105
+ source_property = suggestion_info["property"]
106
+ source_node = suggestion_info["node"]
107
+ suggested_docs = suggestion_info["matches"]
108
+
109
+ source_node_name, source_prop_name, source_prop_type, source_prop_desc = (
110
+ source_node.name,
111
+ source_property.name,
112
+ source_property.type,
113
+ source_property.description,
114
+ )
115
+
116
+ source_additional_metadata = {}
117
+ source_additional_metadata["type"] = source_prop_type
118
+
119
+ for single_suggested_doc, similarity in suggested_docs:
120
+ target_text = single_suggested_doc.page_content
121
+
122
+ (
123
+ target_node_name,
124
+ target_prop_name,
125
+ target_prop_type,
126
+ target_prop_desc,
127
+ ) = get_node_prop_type_desc_from_string(target_text)
128
+
129
+ target_additional_metadata = {}
130
+ target_additional_metadata["type"] = target_prop_type
131
+
132
+ single_suggestion = SingleHarmonizationSuggestion(
133
+ source_node=source_node_name,
134
+ source_property=source_prop_name,
135
+ source_description=source_prop_desc,
136
+ source_additional_metadata=source_additional_metadata,
137
+ target_node=target_node_name,
138
+ target_property=target_prop_name,
139
+ target_description=target_prop_desc,
140
+ target_additional_metadata=target_additional_metadata,
141
+ similarity=similarity,
142
+ )
143
+ suggestions.append(single_suggestion)
144
+
145
+ return HarmonizationSuggestions(suggestions=suggestions)
146
+
147
+ def _get_suggestions_for_source_model(
148
+ self, input_source_model: SimpleDataModel, **kwargs
149
+ ) -> List[SuggestionInfo]:
150
+ suggestions_for_output_model = []
151
+ for node in input_source_model.nodes:
152
+ for node_property in node.properties:
153
+ suggestion_info = {
154
+ "node": node,
155
+ "property": node_property,
156
+ "matches": [],
157
+ }
158
+
159
+ source_query = get_node_property_as_string(node, node_property)
160
+ matches = get_similar_documents(
161
+ self.vectorstore,
162
+ source_query,
163
+ **kwargs,
164
+ )
165
+
166
+ suggestion_info["matches"] = matches
167
+
168
+ if matches:
169
+ suggestions_for_output_model.append(suggestion_info)
170
+
171
+ if not suggestions_for_output_model:
172
+ raise Exception(f"Cannot parse node properties")
173
+
174
+ return suggestions_for_output_model