ai_harmonization 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_harmonization/__init__.py +0 -0
- ai_harmonization/cde.py +644 -0
- ai_harmonization/harmonization_approaches/__init__.py +0 -0
- ai_harmonization/harmonization_approaches/base.py +202 -0
- ai_harmonization/harmonization_approaches/embeddings.py +97 -0
- ai_harmonization/harmonization_approaches/similarity_inmem.py +174 -0
- ai_harmonization/harmonization_benchmark.py +248 -0
- ai_harmonization/interactive.py +69 -0
- ai_harmonization/jsonl.py +197 -0
- ai_harmonization/main.py +12 -0
- ai_harmonization/simple_data_model.py +433 -0
- ai_harmonization/utils.py +471 -0
- ai_harmonization-0.2.6.dist-info/METADATA +159 -0
- ai_harmonization-0.2.6.dist-info/RECORD +17 -0
- ai_harmonization-0.2.6.dist-info/WHEEL +5 -0
- ai_harmonization-0.2.6.dist-info/licenses/LICENSE +201 -0
- ai_harmonization-0.2.6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import io
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from typing import Callable, Dict
|
|
8
|
+
|
|
9
|
+
from ai_harmonization.harmonization_approaches.base import (
|
|
10
|
+
HarmonizationApproach,
|
|
11
|
+
HarmonizationSuggestions,
|
|
12
|
+
SingleHarmonizationSuggestion,
|
|
13
|
+
harmonization_suggestions_to_sssom,
|
|
14
|
+
)
|
|
15
|
+
from ai_harmonization.simple_data_model import (
|
|
16
|
+
SimpleDataModel,
|
|
17
|
+
get_node_prop_type_desc_from_string,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_metrics_for_approach(
|
|
22
|
+
benchmark_filepath: str,
|
|
23
|
+
harmonization_approach: HarmonizationApproach,
|
|
24
|
+
output_filename: str | None = None,
|
|
25
|
+
metrics_column_name: str = "custom_metrics",
|
|
26
|
+
output_sssom_per_row: bool = False,
|
|
27
|
+
output_tsvs_per_row: bool = False,
|
|
28
|
+
output_expected_results_per_row: bool = False,
|
|
29
|
+
source_model_to_simple_model_converter_function: (
|
|
30
|
+
Callable[[str], SimpleDataModel] | None
|
|
31
|
+
) = None,
|
|
32
|
+
target_model_to_simple_model_converter_function: (
|
|
33
|
+
Callable[[str], SimpleDataModel] | None
|
|
34
|
+
) = None,
|
|
35
|
+
**kwargs,
|
|
36
|
+
) -> str:
|
|
37
|
+
"""
|
|
38
|
+
Calculates and outputs metrics for a harmonization approach on a benchmark.
|
|
39
|
+
|
|
40
|
+
This function processes a benchmark file, applies a given harmonization
|
|
41
|
+
approach, and outputs various metrics, including accuracy, precision, recall,
|
|
42
|
+
and F1-score. It also provides options to output SSSOM (Simple Semantic
|
|
43
|
+
Similarity Output Measure) and TSV (Tab-Separated Values) files for each row
|
|
44
|
+
of the benchmark.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
benchmark_filepath: Path to the benchmark file (JSON Lines format).
|
|
48
|
+
harmonization_approach: The harmonization approach to evaluate.
|
|
49
|
+
output_filename: Path to the output file for metrics (JSON Lines format).
|
|
50
|
+
If None, a default filename is generated based on the benchmark file.
|
|
51
|
+
metrics_column_name: The name of the column to store metrics in the
|
|
52
|
+
output file.
|
|
53
|
+
output_sssom_per_row: Whether to output SSSOM files for each row.
|
|
54
|
+
output_tsvs_per_row: Whether to output TSV files for each row.
|
|
55
|
+
output_expected_results_per_row: Whether to output expected results as TSV
|
|
56
|
+
files for each row.
|
|
57
|
+
source_model_to_simple_model_converter_function: A function to convert the
|
|
58
|
+
source model to a SimpleDataModel. If None, a default converter is used.
|
|
59
|
+
target_model_to_simple_model_converter_function: A function to convert the
|
|
60
|
+
target model to a SimpleDataModel. If None, a default converter is used.
|
|
61
|
+
**kwargs: Additional keyword arguments to pass to the harmonization approach.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
The path to the output file containing the metrics.
|
|
65
|
+
"""
|
|
66
|
+
source_model_to_simple_model_converter_function = (
|
|
67
|
+
source_model_to_simple_model_converter_function
|
|
68
|
+
or SimpleDataModel.get_from_unknown_json_format
|
|
69
|
+
)
|
|
70
|
+
target_model_to_simple_model_converter_function = (
|
|
71
|
+
target_model_to_simple_model_converter_function
|
|
72
|
+
or SimpleDataModel.get_from_unknown_json_format
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
if not output_filename:
|
|
76
|
+
dir_name = os.path.dirname(benchmark_filepath)
|
|
77
|
+
base_name = os.path.basename(benchmark_filepath)
|
|
78
|
+
output_filename = os.path.abspath(
|
|
79
|
+
os.path.join(dir_name, "metrics_" + base_name)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
dir_name = os.path.dirname(output_filename)
|
|
83
|
+
sssom_output_dir_name = os.path.join(dir_name, "sssom_per_row")
|
|
84
|
+
tsvs_output_dir_name = os.path.join(dir_name, "sssom_clean_tsvs_per_row")
|
|
85
|
+
expected_results_output_dir_name = os.path.join(
|
|
86
|
+
dir_name, "expected_results_per_row"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
with open(output_filename, "w") as output_file:
|
|
90
|
+
with open(benchmark_filepath, "r", encoding="utf-8") as input_file:
|
|
91
|
+
for i, line in enumerate(input_file):
|
|
92
|
+
row = json.loads(line)
|
|
93
|
+
try:
|
|
94
|
+
input_source_model = json.loads(row["input_source_model"])
|
|
95
|
+
except Exception:
|
|
96
|
+
input_source_model = row["input_source_model"]
|
|
97
|
+
|
|
98
|
+
input_source_model = source_model_to_simple_model_converter_function(
|
|
99
|
+
json.dumps(input_source_model)
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
input_target_model = json.loads(row["input_target_model"])
|
|
104
|
+
except Exception:
|
|
105
|
+
input_target_model = row["input_target_model"]
|
|
106
|
+
|
|
107
|
+
input_target_model = target_model_to_simple_model_converter_function(
|
|
108
|
+
json.dumps(input_target_model)
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
harmonized_mapping = row["harmonized_mapping"]
|
|
112
|
+
expected_harmonization_suggestions = (
|
|
113
|
+
get_harmonization_suggestions_from_harmonized_mapping(
|
|
114
|
+
harmonized_mapping
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
if output_expected_results_per_row:
|
|
118
|
+
filename = os.path.join(
|
|
119
|
+
expected_results_output_dir_name, f"row_{i}.sssom.tsv"
|
|
120
|
+
)
|
|
121
|
+
harmonization_suggestions_to_sssom(
|
|
122
|
+
expected_harmonization_suggestions.suggestions,
|
|
123
|
+
filename=filename,
|
|
124
|
+
exclude_required_comments=True,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
suggestions = harmonization_approach.get_harmonization_suggestions(
|
|
128
|
+
input_source_model=input_source_model,
|
|
129
|
+
input_target_model=input_target_model,
|
|
130
|
+
**kwargs,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
# output standard SSSOM per row
|
|
134
|
+
if output_sssom_per_row:
|
|
135
|
+
filename = os.path.join(sssom_output_dir_name, f"row_{i}.sssom.tsv")
|
|
136
|
+
harmonization_suggestions_to_sssom(
|
|
137
|
+
suggestions.suggestions, filename=filename
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
if output_tsvs_per_row:
|
|
141
|
+
filename = os.path.join(tsvs_output_dir_name, f"row_{i}.sssom.tsv")
|
|
142
|
+
harmonization_suggestions_to_sssom(
|
|
143
|
+
suggestions.suggestions,
|
|
144
|
+
filename=filename,
|
|
145
|
+
exclude_required_comments=True,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
metrics = get_metrics_for_test_case(
|
|
149
|
+
suggestions, expected_harmonization_suggestions
|
|
150
|
+
)
|
|
151
|
+
row[metrics_column_name] = metrics
|
|
152
|
+
line_to_write = json.dumps(row) + "\n"
|
|
153
|
+
output_file.write(line_to_write)
|
|
154
|
+
return output_filename
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def get_harmonization_suggestions_from_harmonized_mapping(
|
|
158
|
+
harmonized_mapping: str,
|
|
159
|
+
) -> HarmonizationSuggestions:
|
|
160
|
+
suggestions = []
|
|
161
|
+
|
|
162
|
+
# Use io.StringIO to treat the string as a file-like object
|
|
163
|
+
tsv_file = io.StringIO(harmonized_mapping)
|
|
164
|
+
reader = csv.reader(tsv_file, delimiter="\t")
|
|
165
|
+
|
|
166
|
+
# Skip the header row
|
|
167
|
+
next(reader)
|
|
168
|
+
|
|
169
|
+
for node_prop_mapping in reader:
|
|
170
|
+
source_model_node_prop_type_desc, target_model_node_prop_type_desc = (
|
|
171
|
+
node_prop_mapping
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
source_node_name, source_prop_name, source_prop_type, source_prop_desc = (
|
|
175
|
+
get_node_prop_type_desc_from_string(source_model_node_prop_type_desc)
|
|
176
|
+
)
|
|
177
|
+
target_node_name, target_prop_name, target_prop_type, target_prop_desc = (
|
|
178
|
+
get_node_prop_type_desc_from_string(target_model_node_prop_type_desc)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
source_additional_metadata = {}
|
|
182
|
+
source_additional_metadata["type"] = source_prop_type
|
|
183
|
+
|
|
184
|
+
target_additional_metadata = {}
|
|
185
|
+
target_additional_metadata["type"] = target_prop_type
|
|
186
|
+
|
|
187
|
+
single_suggestion = SingleHarmonizationSuggestion(
|
|
188
|
+
source_node=source_node_name,
|
|
189
|
+
source_property=source_prop_name,
|
|
190
|
+
source_description=source_prop_desc,
|
|
191
|
+
source_additional_metadata=source_additional_metadata,
|
|
192
|
+
target_node=target_node_name,
|
|
193
|
+
target_property=target_prop_name,
|
|
194
|
+
target_description=target_prop_desc,
|
|
195
|
+
target_additional_metadata=target_additional_metadata,
|
|
196
|
+
)
|
|
197
|
+
suggestions.append(single_suggestion)
|
|
198
|
+
|
|
199
|
+
return HarmonizationSuggestions(suggestions=suggestions)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def get_metrics_for_test_case(
|
|
203
|
+
suggestions: HarmonizationSuggestions, expected_mappings: HarmonizationSuggestions
|
|
204
|
+
) -> Dict:
|
|
205
|
+
HarmonizationSuggestions.model_validate(suggestions)
|
|
206
|
+
HarmonizationSuggestions.model_validate(expected_mappings)
|
|
207
|
+
|
|
208
|
+
def to_pair(s: SingleHarmonizationSuggestion):
|
|
209
|
+
return (s.source_node, s.source_property, s.target_node, s.target_property)
|
|
210
|
+
|
|
211
|
+
suggestion_pairs = set(to_pair(s) for s in suggestions.suggestions)
|
|
212
|
+
expected_pairs = set(to_pair(e) for e in expected_mappings.suggestions)
|
|
213
|
+
|
|
214
|
+
correct_pairs = suggestion_pairs & expected_pairs
|
|
215
|
+
missing_pairs = expected_pairs - suggestion_pairs
|
|
216
|
+
missing_mappings = [
|
|
217
|
+
f"{pair[0]}.{pair[1]} -> {pair[2]}.{pair[3]}" for pair in missing_pairs
|
|
218
|
+
]
|
|
219
|
+
|
|
220
|
+
n_correct = len(correct_pairs)
|
|
221
|
+
n_suggested = len(suggestion_pairs)
|
|
222
|
+
n_expected = len(expected_pairs)
|
|
223
|
+
|
|
224
|
+
# (node.property -> node.property) accuracy
|
|
225
|
+
# e.g. recall
|
|
226
|
+
accuracy = n_correct / n_expected if n_expected else 0.0
|
|
227
|
+
|
|
228
|
+
# how many suggestions were correct
|
|
229
|
+
precision = n_correct / n_suggested if n_suggested else 0.0
|
|
230
|
+
|
|
231
|
+
# how many expected were found
|
|
232
|
+
recall = n_correct / n_expected if n_expected else 0.0
|
|
233
|
+
|
|
234
|
+
if precision + recall > 0:
|
|
235
|
+
f1 = 2 * (precision * recall) / (precision + recall)
|
|
236
|
+
else:
|
|
237
|
+
f1 = 0.0
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
"n_suggested": n_suggested,
|
|
241
|
+
"n_expected": n_expected,
|
|
242
|
+
"n_correct": n_correct,
|
|
243
|
+
"overall_accuracy": accuracy,
|
|
244
|
+
"precision": precision,
|
|
245
|
+
"recall": recall,
|
|
246
|
+
"f1_score": f1,
|
|
247
|
+
"missing_mappings": missing_mappings,
|
|
248
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from IPython.display import display
|
|
3
|
+
from itables.widget import ITable
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_interactive_table_for_suggestions(
|
|
7
|
+
suggestions_df, column_for_filtering=1, table_description=None, **kwargs
|
|
8
|
+
):
|
|
9
|
+
table_description = (
|
|
10
|
+
table_description
|
|
11
|
+
or """
|
|
12
|
+
This table allows you to:
|
|
13
|
+
1. Filter for each "Original Node.Property"
|
|
14
|
+
2. Make a selection from the table below
|
|
15
|
+
(which contains the suggestions based on CDEs for the selected "Original Node.Property")
|
|
16
|
+
3. Multi-select (a ctrl / cmd click will add and remove items from the selection)
|
|
17
|
+
"""
|
|
18
|
+
)
|
|
19
|
+
table = ITable(
|
|
20
|
+
suggestions_df,
|
|
21
|
+
select=True,
|
|
22
|
+
buttons=[
|
|
23
|
+
{
|
|
24
|
+
"extend": "colvis",
|
|
25
|
+
"collectionLayout": "fixed columns",
|
|
26
|
+
"popoverTitle": "Column visibility control",
|
|
27
|
+
}
|
|
28
|
+
], # "copyHtml5", "csvHtml5", "excelHtml5"
|
|
29
|
+
layout={"top1": "searchPanes"},
|
|
30
|
+
searchPanes={
|
|
31
|
+
"layout": "columns-1",
|
|
32
|
+
"cascadePanes": True,
|
|
33
|
+
"columns": [column_for_filtering],
|
|
34
|
+
},
|
|
35
|
+
# showIndex=False,
|
|
36
|
+
columnDefs=[{"className": "dt-left", "targets": "_all"}],
|
|
37
|
+
# classes="display nowrap table_with_monospace_font",
|
|
38
|
+
allow_html=True,
|
|
39
|
+
**kwargs,
|
|
40
|
+
)
|
|
41
|
+
print(table_description)
|
|
42
|
+
return table
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_nodes_and_properties_df(
|
|
46
|
+
data_model_json: dict,
|
|
47
|
+
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
48
|
+
"""
|
|
49
|
+
This function takes a dictionary containing data model information and returns two pandas DataFrames.
|
|
50
|
+
The first DataFrame contains nodes with their names and descriptions, and the second DataFrame contains
|
|
51
|
+
properties of these nodes.
|
|
52
|
+
|
|
53
|
+
Parameters:
|
|
54
|
+
data_model_json (dict): A dictionary containing data model information.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
tuple: Two pandas DataFrames - one for nodes and another for properties.
|
|
58
|
+
"""
|
|
59
|
+
nodes = []
|
|
60
|
+
properties = []
|
|
61
|
+
for node in data_model_json["nodes"]:
|
|
62
|
+
nodes.append({"Name": node["name"], "Description": node["description"]})
|
|
63
|
+
for prop in node["properties"]:
|
|
64
|
+
properties.append({"Node Name": node["name"], **prop})
|
|
65
|
+
|
|
66
|
+
nodes_df = pd.DataFrame(nodes)
|
|
67
|
+
properties_df = pd.DataFrame(properties)
|
|
68
|
+
|
|
69
|
+
return nodes_df, properties_df
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def split_randomize_jsonl_file(input_filename, output_filenames):
|
|
10
|
+
"""
|
|
11
|
+
Opens a JSONL file, reads its contents, randomizes the order of the lines,
|
|
12
|
+
and writes them out to multiple files with specified percentages.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
input_filename (str): The path to the JSONL file to be randomized.
|
|
16
|
+
output_filenames (list): A list of tuples containing the filename and percentage for each output file.
|
|
17
|
+
|
|
18
|
+
Example usage:
|
|
19
|
+
split_randomize_jsonl_file('input.jsonl', [('output1.jsonl', 40), ('validation.jsonl', 30), ('test_output.jsonl', 30)])
|
|
20
|
+
"""
|
|
21
|
+
with open(input_filename, "r") as input_file:
|
|
22
|
+
data = [line for line in input_file]
|
|
23
|
+
|
|
24
|
+
if not data:
|
|
25
|
+
for output_filename, _ in output_filenames:
|
|
26
|
+
with open(output_filename, "w"):
|
|
27
|
+
logging.warning(f"Writing nothing to {output_filename}.")
|
|
28
|
+
return
|
|
29
|
+
|
|
30
|
+
# Randomize the order of the lines
|
|
31
|
+
logging.info(f"Shuffling {len(data)} lines from {input_filename}")
|
|
32
|
+
random.shuffle(data)
|
|
33
|
+
|
|
34
|
+
total_percentage = sum(percentage for _, percentage in output_filenames)
|
|
35
|
+
assert round(total_percentage, 6) == 100.0
|
|
36
|
+
|
|
37
|
+
i = 0
|
|
38
|
+
percentage_to_get_to = 0
|
|
39
|
+
for output_filename, percentage in output_filenames:
|
|
40
|
+
lines_in_file = 0
|
|
41
|
+
percentage_to_get_to += percentage
|
|
42
|
+
with open(output_filename, "w") as output_file:
|
|
43
|
+
# while the percentage of processed lines is less than the target percentage, continue
|
|
44
|
+
# in case there's not an even split, ensure we don't go over the size of the overall data
|
|
45
|
+
while ((float(i) / len(data)) * 100) <= percentage_to_get_to and i < len(
|
|
46
|
+
data
|
|
47
|
+
):
|
|
48
|
+
output_file.write(data[i])
|
|
49
|
+
i += 1
|
|
50
|
+
lines_in_file += 1
|
|
51
|
+
|
|
52
|
+
if percentage_to_get_to == 100:
|
|
53
|
+
# we can miss 1 line above due to rounding, so add it here
|
|
54
|
+
i -= 1
|
|
55
|
+
while i < len(data):
|
|
56
|
+
output_file.write(data[i])
|
|
57
|
+
i += 1
|
|
58
|
+
logging.info(
|
|
59
|
+
f"Wrote {lines_in_file} lines of total {len(data)} to {output_filename} to meet percentage {percentage}."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def split_jsonl_into_separate_files(input_filepath, output_directory):
|
|
64
|
+
"""
|
|
65
|
+
Reads a JSONL file and saves each line as a separate JSONL file
|
|
66
|
+
in the specified output directory.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
input_filepath (str): The path to the input JSONL file.
|
|
70
|
+
output_directory (str): The directory where the output files will be saved.
|
|
71
|
+
"""
|
|
72
|
+
import json
|
|
73
|
+
import os
|
|
74
|
+
|
|
75
|
+
with open(input_filepath, "r") as f:
|
|
76
|
+
i = 0
|
|
77
|
+
for item in f:
|
|
78
|
+
output_filepath = os.path.join(output_directory, f"output_{i}.jsonl")
|
|
79
|
+
with open(
|
|
80
|
+
output_filepath,
|
|
81
|
+
"w",
|
|
82
|
+
) as f2:
|
|
83
|
+
f2.write(item)
|
|
84
|
+
i += 1
|
|
85
|
+
|
|
86
|
+
# Print a message to indicate completion
|
|
87
|
+
print("JSONL data successfully converted and saved to separate files.")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def filter_jsonl_by_length(input_file, max_length, output_file=None):
|
|
91
|
+
"""
|
|
92
|
+
Filters a JSONL file, keeping only lines below a specified character limit.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
input_file: Path to the input JSONL file.
|
|
96
|
+
max_length: The maximum character length for a line to be included.
|
|
97
|
+
output_file: (Optional) Path to the output file. If None,
|
|
98
|
+
a default filename is created based on the input filename.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
None. Creates an output file with the filtered lines. Raises
|
|
102
|
+
FileNotFoundError if the input file doesn't exist. Raises
|
|
103
|
+
ValueError if max_length is not a positive integer.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
if not os.path.exists(input_file):
|
|
107
|
+
raise FileNotFoundError(f"Input file not found: {input_file}")
|
|
108
|
+
|
|
109
|
+
if not isinstance(max_length, int) or max_length <= 0:
|
|
110
|
+
raise ValueError("max_length must be a positive integer.")
|
|
111
|
+
|
|
112
|
+
output_dir = "output"
|
|
113
|
+
if output_file is None:
|
|
114
|
+
# Create a default output filename in the same directory
|
|
115
|
+
input_dir, input_filename = os.path.split(input_file)
|
|
116
|
+
base_name, ext = os.path.splitext(input_filename)
|
|
117
|
+
output_file = os.path.join(input_dir, f"{base_name}_filtered{ext}")
|
|
118
|
+
else:
|
|
119
|
+
# get path to output file:
|
|
120
|
+
output_dir = os.path.dirname(output_file)
|
|
121
|
+
|
|
122
|
+
# Check if the output directory exists, create it if it doesn't
|
|
123
|
+
if not os.path.exists(output_dir):
|
|
124
|
+
os.makedirs(output_dir)
|
|
125
|
+
|
|
126
|
+
with (
|
|
127
|
+
open(input_file, "r", encoding="utf-8") as infile,
|
|
128
|
+
open(output_file, "w", encoding="utf-8") as outfile,
|
|
129
|
+
):
|
|
130
|
+
for line in infile:
|
|
131
|
+
if (
|
|
132
|
+
len(line.strip()) < max_length
|
|
133
|
+
): # strip() removes leading/trailing whitespace
|
|
134
|
+
outfile.write(line)
|
|
135
|
+
logging.info(f"Filtered file written to: {output_file}")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def jsonl_to_csv(jsonl_path, csv_path):
|
|
139
|
+
records = []
|
|
140
|
+
with open(jsonl_path, "r", encoding="utf-8") as f:
|
|
141
|
+
for i, line in enumerate(f):
|
|
142
|
+
# Skip empty lines
|
|
143
|
+
if not line.strip():
|
|
144
|
+
continue
|
|
145
|
+
try:
|
|
146
|
+
obj = json.loads(line)
|
|
147
|
+
records.append(obj)
|
|
148
|
+
except json.JSONDecodeError as e:
|
|
149
|
+
print(f"Warning: JSON decode error on line {i+1}: {e}")
|
|
150
|
+
print(line)
|
|
151
|
+
continue
|
|
152
|
+
|
|
153
|
+
if not records:
|
|
154
|
+
print("No data found!")
|
|
155
|
+
return
|
|
156
|
+
|
|
157
|
+
headers = records[0].keys()
|
|
158
|
+
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
|
159
|
+
writer = csv.DictWriter(f, fieldnames=headers)
|
|
160
|
+
writer.writeheader()
|
|
161
|
+
writer.writerows(records)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def split_harmonization_jsonl_by_input_target_model(jsonl_path, output_dir):
|
|
165
|
+
"""
|
|
166
|
+
Splits the JSONL into one file per unique input_target_model,
|
|
167
|
+
using a SHA256 checksum of the input_target_model string as the filename.
|
|
168
|
+
"""
|
|
169
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
170
|
+
|
|
171
|
+
# Map from input_target_model value to its checksum filename
|
|
172
|
+
model_to_filename = {}
|
|
173
|
+
grouped_rows = {}
|
|
174
|
+
|
|
175
|
+
with open(jsonl_path, "r", encoding="utf-8") as infile:
|
|
176
|
+
for line in infile:
|
|
177
|
+
row = json.loads(line)
|
|
178
|
+
key = str(row["input_target_model"])
|
|
179
|
+
# Compute SHA256 checksum of the input_target_model string
|
|
180
|
+
checksum = hashlib.sha256(key.encode("utf-8")).hexdigest()
|
|
181
|
+
filename = f"target_model_{checksum}"
|
|
182
|
+
if checksum not in model_to_filename:
|
|
183
|
+
model_to_filename[checksum] = filename
|
|
184
|
+
grouped_rows[filename] = []
|
|
185
|
+
grouped_rows[filename].append(row)
|
|
186
|
+
|
|
187
|
+
# Write out each group
|
|
188
|
+
for filename, rows in grouped_rows.items():
|
|
189
|
+
with open(
|
|
190
|
+
os.path.join(output_dir, f"{filename}.jsonl"), "w", encoding="utf-8"
|
|
191
|
+
) as outfile:
|
|
192
|
+
for row in rows:
|
|
193
|
+
json.dump(row, outfile)
|
|
194
|
+
outfile.write("\n")
|
|
195
|
+
print(
|
|
196
|
+
f"Wrote {len(model_to_filename)} JSONL files (one per unique input_target_model) to: {output_dir}"
|
|
197
|
+
)
|
ai_harmonization/main.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
from ai_harmonization.harmonization_approaches.similarity_inmem import (
|
|
6
|
+
SimilaritySearchInMemoryVectorDb,
|
|
7
|
+
)
|
|
8
|
+
from ai_harmonization.harmonization_benchmark import get_metrics_for_approach
|
|
9
|
+
from ai_harmonization.jsonl import (
|
|
10
|
+
jsonl_to_csv,
|
|
11
|
+
split_harmonization_jsonl_by_input_target_model,
|
|
12
|
+
)
|