ai_harmonization 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_harmonization/__init__.py +0 -0
- ai_harmonization/cde.py +644 -0
- ai_harmonization/harmonization_approaches/__init__.py +0 -0
- ai_harmonization/harmonization_approaches/base.py +202 -0
- ai_harmonization/harmonization_approaches/embeddings.py +97 -0
- ai_harmonization/harmonization_approaches/similarity_inmem.py +174 -0
- ai_harmonization/harmonization_benchmark.py +248 -0
- ai_harmonization/interactive.py +69 -0
- ai_harmonization/jsonl.py +197 -0
- ai_harmonization/main.py +12 -0
- ai_harmonization/simple_data_model.py +433 -0
- ai_harmonization/utils.py +471 -0
- ai_harmonization-0.2.6.dist-info/METADATA +159 -0
- ai_harmonization-0.2.6.dist-info/RECORD +17 -0
- ai_harmonization-0.2.6.dist-info/WHEEL +5 -0
- ai_harmonization-0.2.6.dist-info/licenses/LICENSE +201 -0
- ai_harmonization-0.2.6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from typing import Dict, List, Optional
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from sssom.parsers import MappingSetDataFrame
|
|
8
|
+
from sssom.writers import write_table
|
|
9
|
+
from sssom_schema import Mapping, MappingSet
|
|
10
|
+
|
|
11
|
+
from ai_harmonization.simple_data_model import SimpleDataModel
|
|
12
|
+
from ai_harmonization.utils import nodeproperty_to_curie
|
|
13
|
+
|
|
14
|
+
DEFAULT_SSSOM_METADATA: Dict[str, str | dict] = {
|
|
15
|
+
"mapping_set_id": "https://example.org/mappings/example.sssom.tsv",
|
|
16
|
+
"license": "https://www.apache.org/licenses/LICENSE-2.0",
|
|
17
|
+
"mapping_set_description": (
|
|
18
|
+
"This mapping set aligns properties (columns/fields) between source and target data models (graph or relational). "
|
|
19
|
+
"For each mapping, subject_id and object_id are composite identifiers in the form 'node.property', "
|
|
20
|
+
"where 'node' is the original entity/table/class, and 'property' is the attribute/field/column. "
|
|
21
|
+
"For example, 'Subject.name' refers to the 'name' property of the 'Subject' node in the source model."
|
|
22
|
+
),
|
|
23
|
+
}
|
|
24
|
+
SSSOM_PREFIX_MAP = {"EX": "http://example.org/"}
|
|
25
|
+
DEFAULT_SSSOM_PREDICATE_ID: str = "skos:relatedMatch"
|
|
26
|
+
DEFAULT_SSSOM_MAPPING_JUSTIFICATION: str = "semapv:UnspecifiedMatching"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SingleHarmonizationSuggestion(BaseModel):
|
|
30
|
+
"""
|
|
31
|
+
Represents a harmonization suggestion for a pair of node.property attributes
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
source_node: str
|
|
35
|
+
source_property: str
|
|
36
|
+
source_description: str
|
|
37
|
+
source_additional_metadata: Optional[dict] = None
|
|
38
|
+
target_node: str
|
|
39
|
+
target_property: str
|
|
40
|
+
target_description: str
|
|
41
|
+
target_additional_metadata: Optional[dict] = None
|
|
42
|
+
similarity: Optional[float] = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class HarmonizationSuggestions(BaseModel):
|
|
46
|
+
"""
|
|
47
|
+
Container for a batch of harmonization suggestions.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
suggestions: List[SingleHarmonizationSuggestion]
|
|
51
|
+
|
|
52
|
+
def to_dataframe(self) -> pd.DataFrame:
|
|
53
|
+
return pd.DataFrame([suggestion.__dict__ for suggestion in self.suggestions])
|
|
54
|
+
|
|
55
|
+
def to_simlified_dataframe(self) -> pd.DataFrame:
|
|
56
|
+
data = []
|
|
57
|
+
for suggestion in self.suggestions:
|
|
58
|
+
data.append(
|
|
59
|
+
{
|
|
60
|
+
"Original Node.Property": f"{suggestion.source_node}.{suggestion.source_property}",
|
|
61
|
+
"Suggested Target Node.Property": f"{suggestion.target_node}.{suggestion.target_property}",
|
|
62
|
+
"Similarity": suggestion.similarity,
|
|
63
|
+
"Target Description": suggestion.target_description,
|
|
64
|
+
"Original Description": suggestion.source_description,
|
|
65
|
+
}
|
|
66
|
+
)
|
|
67
|
+
return pd.DataFrame(data)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class HarmonizationApproach(ABC):
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def get_harmonization_suggestions(
|
|
73
|
+
self,
|
|
74
|
+
input_source_model: SimpleDataModel,
|
|
75
|
+
input_target_model: SimpleDataModel,
|
|
76
|
+
**kwargs,
|
|
77
|
+
) -> HarmonizationSuggestions:
|
|
78
|
+
"""
|
|
79
|
+
Returns HarmonizationSuggestions according to the implementing algorithm.
|
|
80
|
+
"""
|
|
81
|
+
raise NotImplementedError()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class ExampleHarmonizationApproach(HarmonizationApproach):
|
|
85
|
+
"""
|
|
86
|
+
Dummy example that returns a static harmonization suggestion.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def get_harmonization_suggestions(
|
|
90
|
+
self,
|
|
91
|
+
input_source_model: SimpleDataModel,
|
|
92
|
+
input_target_model: SimpleDataModel,
|
|
93
|
+
**kwargs,
|
|
94
|
+
) -> HarmonizationSuggestions:
|
|
95
|
+
"""
|
|
96
|
+
Example implementation for demonstration.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
suggestions: List[SingleHarmonizationSuggestion] = [
|
|
100
|
+
SingleHarmonizationSuggestion(
|
|
101
|
+
source_node="source_node_1",
|
|
102
|
+
source_property="source_property_1",
|
|
103
|
+
source_description="source_description_1",
|
|
104
|
+
source_additional_metadata={"type": "string"},
|
|
105
|
+
target_node="target_node_1",
|
|
106
|
+
target_property="target_property_1",
|
|
107
|
+
target_description="target_description_1",
|
|
108
|
+
target_additional_metadata={"type": "number"},
|
|
109
|
+
similarity=0.9,
|
|
110
|
+
)
|
|
111
|
+
]
|
|
112
|
+
return HarmonizationSuggestions(suggestions=suggestions)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def harmonization_suggestions_to_sssom(
|
|
116
|
+
suggestions: List[SingleHarmonizationSuggestion],
|
|
117
|
+
filename: str,
|
|
118
|
+
meta: Optional[dict] = None,
|
|
119
|
+
exclude_required_comments: bool = False,
|
|
120
|
+
) -> None:
|
|
121
|
+
"""
|
|
122
|
+
Convert a list of harmonization suggestions into a MappingSet and write to a SSSOM TSV file.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
suggestions: List of SingleHarmonizationSuggestion.
|
|
126
|
+
filename: Output .sssom.tsv file.
|
|
127
|
+
meta: Optional SSSOM MappingSet metadata dictionary. If None, uses DEFAULT_SSSOM_METADATA.
|
|
128
|
+
exclude_required_comments: Will output a non-standard SSSOM file with no comments. This allows
|
|
129
|
+
you to create a "clean" TSV which starts with headers and contains only rows after
|
|
130
|
+
"""
|
|
131
|
+
meta = meta or DEFAULT_SSSOM_METADATA
|
|
132
|
+
mappings = [
|
|
133
|
+
suggestion_to_mapping(single_suggestion) for single_suggestion in suggestions
|
|
134
|
+
]
|
|
135
|
+
mapping_set = MappingSet(**meta, mappings=mappings)
|
|
136
|
+
write_mappingset_to_sssom(mapping_set, filename)
|
|
137
|
+
|
|
138
|
+
if exclude_required_comments:
|
|
139
|
+
with open(filename, "r") as file:
|
|
140
|
+
lines = [line for line in file if not line.lstrip().startswith("#")]
|
|
141
|
+
with open(filename, "w") as file:
|
|
142
|
+
file.writelines(lines)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def suggestion_to_mapping(
|
|
146
|
+
suggestion: SingleHarmonizationSuggestion,
|
|
147
|
+
predicate_id: Optional[str] = None,
|
|
148
|
+
mapping_justification: Optional[str] = None,
|
|
149
|
+
) -> Mapping:
|
|
150
|
+
"""
|
|
151
|
+
Convert a SingleHarmonizationSuggestion to an SSSOM Mapping.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
suggestion: The harmonization suggestion to convert.
|
|
155
|
+
predicate_id: Which predicate to use; default is DEFAULT_SSSOM_PREDICATE_ID.
|
|
156
|
+
mapping_justification: Which mapping justification to use; default is DEFAULT_SSSOM_MAPPING_JUSTIFICATION.
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
Mapping: The equivalent SSSOM Mapping object.
|
|
160
|
+
"""
|
|
161
|
+
predicate_id = predicate_id or DEFAULT_SSSOM_PREDICATE_ID
|
|
162
|
+
mapping_justification = mapping_justification or DEFAULT_SSSOM_MAPPING_JUSTIFICATION
|
|
163
|
+
|
|
164
|
+
# SSSOM requires CURIEs or URLs.
|
|
165
|
+
subject_id = nodeproperty_to_curie(
|
|
166
|
+
f"{suggestion.source_node}.{suggestion.source_property}"
|
|
167
|
+
)
|
|
168
|
+
object_id = nodeproperty_to_curie(
|
|
169
|
+
f"{suggestion.target_node}.{suggestion.target_property}"
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
return Mapping(
|
|
173
|
+
predicate_id=predicate_id,
|
|
174
|
+
mapping_justification=mapping_justification,
|
|
175
|
+
subject_id=subject_id,
|
|
176
|
+
subject_label=suggestion.source_description,
|
|
177
|
+
object_id=object_id,
|
|
178
|
+
object_label=suggestion.target_description,
|
|
179
|
+
object_source=None, # TODO: data dictionary. note there's also a version
|
|
180
|
+
similarity_score=suggestion.similarity,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def write_mappingset_to_sssom(
|
|
185
|
+
mapping_set: MappingSet,
|
|
186
|
+
filename: str,
|
|
187
|
+
) -> None:
|
|
188
|
+
"""
|
|
189
|
+
Write a MappingSet object to a SSSOM TSV file.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
mapping_set: The SSSOM MappingSet to write.
|
|
193
|
+
filename: The path to write the .sssom.tsv file.
|
|
194
|
+
"""
|
|
195
|
+
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
|
196
|
+
|
|
197
|
+
mapping_set_df = MappingSetDataFrame.from_mapping_set(
|
|
198
|
+
mapping_set, converter=SSSOM_PREFIX_MAP
|
|
199
|
+
)
|
|
200
|
+
mapping_set_df.clean_prefix_map()
|
|
201
|
+
with open(filename, "w", encoding="utf-8") as output_file:
|
|
202
|
+
write_table(mapping_set_df, output_file)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import torch
|
|
3
|
+
from transformers import AutoModel, AutoTokenizer
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BaseEmbeddings:
|
|
7
|
+
def __init__(self, model_name, device=None, trust_remote_code=False):
|
|
8
|
+
# Load tokenizer/model, set device, set eval()
|
|
9
|
+
self.tokenizer = AutoTokenizer.from_pretrained(
|
|
10
|
+
model_name, trust_remote_code=trust_remote_code
|
|
11
|
+
)
|
|
12
|
+
self.device = torch.device(
|
|
13
|
+
device if device else ("cuda" if torch.cuda.is_available() else "cpu")
|
|
14
|
+
)
|
|
15
|
+
self.model = (
|
|
16
|
+
AutoModel.from_pretrained(model_name, trust_remote_code=trust_remote_code)
|
|
17
|
+
.to(self.device)
|
|
18
|
+
.eval()
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
def embed_query(self, text):
|
|
22
|
+
return self.embed_documents([text])[0]
|
|
23
|
+
|
|
24
|
+
def mean_pool(self, last_hidden, mask):
|
|
25
|
+
# Shape: (batch, seq_len, dim), mask: (batch, seq_len, 1)
|
|
26
|
+
masked = last_hidden * mask
|
|
27
|
+
summed = masked.sum(1)
|
|
28
|
+
counts = mask.sum(1)
|
|
29
|
+
mean_pooled = summed / counts.clamp(min=1e-9)
|
|
30
|
+
return mean_pooled
|
|
31
|
+
|
|
32
|
+
def embed_documents(self, texts):
|
|
33
|
+
raise NotImplementedError("Implement in subclass.")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class MedGemmaEmbeddings(BaseEmbeddings):
|
|
37
|
+
def __init__(self, model_name="google/medgemma-4b-it", device=None):
|
|
38
|
+
super().__init__(model_name, device, trust_remote_code=True)
|
|
39
|
+
|
|
40
|
+
def embed_documents(self, texts):
|
|
41
|
+
inputs = self.tokenizer(
|
|
42
|
+
texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
|
|
43
|
+
).to(self.device)
|
|
44
|
+
with torch.no_grad():
|
|
45
|
+
outputs = self.model(**inputs, output_hidden_states=True)
|
|
46
|
+
last_hidden = outputs.hidden_states[-1]
|
|
47
|
+
mask = (
|
|
48
|
+
inputs["attention_mask"]
|
|
49
|
+
.unsqueeze(-1)
|
|
50
|
+
.expand(last_hidden.size())
|
|
51
|
+
.float()
|
|
52
|
+
)
|
|
53
|
+
pooled = self.mean_pool(last_hidden, mask)
|
|
54
|
+
return pooled.cpu().numpy().tolist()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class QwenEmbeddings(BaseEmbeddings):
|
|
58
|
+
def __init__(self, model_name="Qwen/Qwen3-Embedding-0.6B", device=None):
|
|
59
|
+
super().__init__(model_name, device, trust_remote_code=True)
|
|
60
|
+
|
|
61
|
+
def embed_documents(self, texts):
|
|
62
|
+
inputs = self.tokenizer(
|
|
63
|
+
texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
|
|
64
|
+
).to(self.device)
|
|
65
|
+
with torch.no_grad():
|
|
66
|
+
outputs = self.model(**inputs, output_hidden_states=True)
|
|
67
|
+
# recommended by Qwen
|
|
68
|
+
pooled = outputs.get("pooled_output", None)
|
|
69
|
+
if pooled is not None:
|
|
70
|
+
return pooled.cpu().numpy().tolist()
|
|
71
|
+
# fallback to mean pool
|
|
72
|
+
last_hidden = outputs.hidden_states[-1]
|
|
73
|
+
mask = (
|
|
74
|
+
inputs["attention_mask"]
|
|
75
|
+
.unsqueeze(-1)
|
|
76
|
+
.expand(last_hidden.size())
|
|
77
|
+
.float()
|
|
78
|
+
)
|
|
79
|
+
pooled = self.mean_pool(last_hidden, mask)
|
|
80
|
+
return pooled.cpu().numpy().tolist()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class BGEEmbeddings(BaseEmbeddings):
|
|
84
|
+
def __init__(self, model_name="BAAI/bge-large-en-v1.5", device=None):
|
|
85
|
+
super().__init__(model_name, device=device)
|
|
86
|
+
|
|
87
|
+
def embed_documents(self, texts):
|
|
88
|
+
inputs = self.tokenizer(
|
|
89
|
+
texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
|
|
90
|
+
).to(self.device)
|
|
91
|
+
with torch.no_grad():
|
|
92
|
+
model_output = self.model(**inputs)
|
|
93
|
+
mask = inputs["attention_mask"].unsqueeze(-1)
|
|
94
|
+
pooled = self.mean_pool(model_output.last_hidden_state, mask)
|
|
95
|
+
pooled = pooled.cpu().numpy()
|
|
96
|
+
pooled = pooled / np.linalg.norm(pooled, axis=1, keepdims=True)
|
|
97
|
+
return pooled.tolist()
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import List, TypedDict
|
|
3
|
+
|
|
4
|
+
from langchain_core.documents import Document
|
|
5
|
+
from langchain_huggingface import HuggingFaceEmbeddings
|
|
6
|
+
|
|
7
|
+
from ai_harmonization.harmonization_approaches.base import (
|
|
8
|
+
HarmonizationApproach,
|
|
9
|
+
HarmonizationSuggestions,
|
|
10
|
+
SingleHarmonizationSuggestion,
|
|
11
|
+
)
|
|
12
|
+
from ai_harmonization.simple_data_model import (
|
|
13
|
+
Node,
|
|
14
|
+
Property,
|
|
15
|
+
SimpleDataModel,
|
|
16
|
+
get_data_model_as_langchain_documents,
|
|
17
|
+
get_node_prop_type_desc_from_string,
|
|
18
|
+
get_node_property_as_string,
|
|
19
|
+
)
|
|
20
|
+
from ai_harmonization.utils import (
|
|
21
|
+
TEMP_DIR,
|
|
22
|
+
add_documents_to_vectorstore,
|
|
23
|
+
get_langchain_vectorstore_and_persistent_client,
|
|
24
|
+
get_similar_documents,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ExistingVectorstoreException(BaseException):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SuggestionInfo(TypedDict):
|
|
33
|
+
node: Node
|
|
34
|
+
property: Property
|
|
35
|
+
matches: List[Document]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SimilaritySearchInMemoryVectorDb(HarmonizationApproach):
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
vectordb_persist_directory_name: str,
|
|
43
|
+
input_target_model: SimpleDataModel,
|
|
44
|
+
embedding_function: HuggingFaceEmbeddings | None = None,
|
|
45
|
+
force_vectorstore_recreation: bool = False,
|
|
46
|
+
batch_size: int | None = None,
|
|
47
|
+
):
|
|
48
|
+
super().__init__()
|
|
49
|
+
|
|
50
|
+
self.embedding_function = embedding_function or HuggingFaceEmbeddings(
|
|
51
|
+
model_name="sentence-transformers/all-mpnet-base-v2"
|
|
52
|
+
)
|
|
53
|
+
self.vectordb_persist_directory_name = vectordb_persist_directory_name
|
|
54
|
+
vectorstore, persistent_client = (
|
|
55
|
+
get_langchain_vectorstore_and_persistent_client(
|
|
56
|
+
persist_directory_name=self.vectordb_persist_directory_name,
|
|
57
|
+
embedding_function=self.embedding_function,
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
self.vectorstore = vectorstore
|
|
61
|
+
self.persistent_client = persistent_client
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
self._add_target_to_vector_database(
|
|
65
|
+
input_target_model,
|
|
66
|
+
force_recreation=force_vectorstore_recreation,
|
|
67
|
+
batch_size=batch_size,
|
|
68
|
+
)
|
|
69
|
+
except ExistingVectorstoreException:
|
|
70
|
+
logging.info("vectorstore already exists, NOT recreating...")
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
def _add_target_to_vector_database(
|
|
74
|
+
self, input_target_model, force_recreation=False, batch_size=None
|
|
75
|
+
):
|
|
76
|
+
if force_recreation:
|
|
77
|
+
all_ids = self.vectorstore.get()["ids"]
|
|
78
|
+
if all_ids:
|
|
79
|
+
self.vectorstore.delete(ids=all_ids)
|
|
80
|
+
|
|
81
|
+
if len(self.vectorstore.get()["ids"]) != 0:
|
|
82
|
+
raise ExistingVectorstoreException(
|
|
83
|
+
f"Vectorstore in persist directory {self.vectordb_persist_directory_name} already has data here: {TEMP_DIR}, "
|
|
84
|
+
"aborting re-adding. Delete the persist directory, use a new one, or force recreation."
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
target_docs = get_data_model_as_langchain_documents(input_target_model)
|
|
88
|
+
|
|
89
|
+
add_documents_to_vectorstore(
|
|
90
|
+
target_docs, self.vectorstore, self.persistent_client, batch_size
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def get_harmonization_suggestions(
|
|
94
|
+
self,
|
|
95
|
+
input_source_model: SimpleDataModel,
|
|
96
|
+
input_target_model: SimpleDataModel,
|
|
97
|
+
**kwargs,
|
|
98
|
+
):
|
|
99
|
+
# note: k and score_threshold are in kwargs
|
|
100
|
+
suggestions_for_output_model = self._get_suggestions_for_source_model(
|
|
101
|
+
input_source_model, **kwargs
|
|
102
|
+
)
|
|
103
|
+
suggestions = []
|
|
104
|
+
for suggestion_info in suggestions_for_output_model:
|
|
105
|
+
source_property = suggestion_info["property"]
|
|
106
|
+
source_node = suggestion_info["node"]
|
|
107
|
+
suggested_docs = suggestion_info["matches"]
|
|
108
|
+
|
|
109
|
+
source_node_name, source_prop_name, source_prop_type, source_prop_desc = (
|
|
110
|
+
source_node.name,
|
|
111
|
+
source_property.name,
|
|
112
|
+
source_property.type,
|
|
113
|
+
source_property.description,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
source_additional_metadata = {}
|
|
117
|
+
source_additional_metadata["type"] = source_prop_type
|
|
118
|
+
|
|
119
|
+
for single_suggested_doc, similarity in suggested_docs:
|
|
120
|
+
target_text = single_suggested_doc.page_content
|
|
121
|
+
|
|
122
|
+
(
|
|
123
|
+
target_node_name,
|
|
124
|
+
target_prop_name,
|
|
125
|
+
target_prop_type,
|
|
126
|
+
target_prop_desc,
|
|
127
|
+
) = get_node_prop_type_desc_from_string(target_text)
|
|
128
|
+
|
|
129
|
+
target_additional_metadata = {}
|
|
130
|
+
target_additional_metadata["type"] = target_prop_type
|
|
131
|
+
|
|
132
|
+
single_suggestion = SingleHarmonizationSuggestion(
|
|
133
|
+
source_node=source_node_name,
|
|
134
|
+
source_property=source_prop_name,
|
|
135
|
+
source_description=source_prop_desc,
|
|
136
|
+
source_additional_metadata=source_additional_metadata,
|
|
137
|
+
target_node=target_node_name,
|
|
138
|
+
target_property=target_prop_name,
|
|
139
|
+
target_description=target_prop_desc,
|
|
140
|
+
target_additional_metadata=target_additional_metadata,
|
|
141
|
+
similarity=similarity,
|
|
142
|
+
)
|
|
143
|
+
suggestions.append(single_suggestion)
|
|
144
|
+
|
|
145
|
+
return HarmonizationSuggestions(suggestions=suggestions)
|
|
146
|
+
|
|
147
|
+
def _get_suggestions_for_source_model(
|
|
148
|
+
self, input_source_model: SimpleDataModel, **kwargs
|
|
149
|
+
) -> List[SuggestionInfo]:
|
|
150
|
+
suggestions_for_output_model = []
|
|
151
|
+
for node in input_source_model.nodes:
|
|
152
|
+
for node_property in node.properties:
|
|
153
|
+
suggestion_info = {
|
|
154
|
+
"node": node,
|
|
155
|
+
"property": node_property,
|
|
156
|
+
"matches": [],
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
source_query = get_node_property_as_string(node, node_property)
|
|
160
|
+
matches = get_similar_documents(
|
|
161
|
+
self.vectorstore,
|
|
162
|
+
source_query,
|
|
163
|
+
**kwargs,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
suggestion_info["matches"] = matches
|
|
167
|
+
|
|
168
|
+
if matches:
|
|
169
|
+
suggestions_for_output_model.append(suggestion_info)
|
|
170
|
+
|
|
171
|
+
if not suggestions_for_output_model:
|
|
172
|
+
raise Exception(f"Cannot parse node properties")
|
|
173
|
+
|
|
174
|
+
return suggestions_for_output_model
|