ai_harmonization 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/PKG-INFO +4 -2
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/embeddings.py +28 -10
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/PKG-INFO +4 -2
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/requires.txt +3 -1
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/pyproject.toml +4 -2
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/LICENSE +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/README.md +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/__init__.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/cde.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/__init__.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/base.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/similarity_inmem.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_benchmark.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/interactive.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/jsonl.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/main.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/simple_data_model.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/utils.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/SOURCES.txt +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/dependency_links.txt +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/top_level.txt +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/setup.cfg +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/tests/test_harmonization.py +0 -0
- {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/tests/test_sssom.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai_harmonization
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: AI-backed harmonization framework
|
|
5
5
|
Requires-Python: <4.0.0,>=3.12
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -13,7 +13,6 @@ Requires-Dist: pydantic>=2.11.7
|
|
|
13
13
|
Requires-Dist: sssom>=0.4.16
|
|
14
14
|
Requires-Dist: sssom-schema>=1.0.0
|
|
15
15
|
Requires-Dist: tqdm>=4.67.1
|
|
16
|
-
Requires-Dist: transformers>=4.57.1
|
|
17
16
|
Requires-Dist: itables>=2.4.4
|
|
18
17
|
Requires-Dist: numpy>=2.0.2
|
|
19
18
|
Requires-Dist: pandas>=2.2.2
|
|
@@ -22,6 +21,9 @@ Requires-Dist: ipython>=7.34.0
|
|
|
22
21
|
Requires-Dist: torch>=2.8.0
|
|
23
22
|
Requires-Dist: matplotlib>=3.10.0
|
|
24
23
|
Requires-Dist: chromadb>=1.0.0
|
|
24
|
+
Requires-Dist: sentence-transformers>=5.1.2
|
|
25
|
+
Requires-Dist: datasets>=4.4.1
|
|
26
|
+
Requires-Dist: accelerate>=1.11.0
|
|
25
27
|
Dynamic: license-file
|
|
26
28
|
|
|
27
29
|
# AI Harmonization
|
|
@@ -1,25 +1,35 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import torch
|
|
3
3
|
from transformers import AutoModel, AutoTokenizer
|
|
4
|
+
from sentence_transformers import SentenceTransformer
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
class BaseEmbeddings:
|
|
7
|
-
def __init__(self, model_name, device=None, trust_remote_code=False):
|
|
8
|
+
def __init__(self, model_name, model=None, device=None, trust_remote_code=False):
|
|
8
9
|
# Load tokenizer/model, set device, set eval()
|
|
9
|
-
|
|
10
|
+
tokenizer = None
|
|
11
|
+
if model:
|
|
12
|
+
tokenizer = model.tokenizer
|
|
13
|
+
|
|
14
|
+
self.model_name = model_name
|
|
15
|
+
|
|
16
|
+
self.tokenizer = tokenizer or AutoTokenizer.from_pretrained(
|
|
10
17
|
model_name, trust_remote_code=trust_remote_code
|
|
11
18
|
)
|
|
12
19
|
self.device = torch.device(
|
|
13
20
|
device if device else ("cuda" if torch.cuda.is_available() else "cpu")
|
|
14
21
|
)
|
|
15
|
-
self.model = (
|
|
22
|
+
self.model = model or (
|
|
16
23
|
AutoModel.from_pretrained(model_name, trust_remote_code=trust_remote_code)
|
|
17
24
|
.to(self.device)
|
|
18
25
|
.eval()
|
|
19
26
|
)
|
|
20
27
|
|
|
21
28
|
def embed_query(self, text):
|
|
22
|
-
|
|
29
|
+
if isinstance(self.model, SentenceTransformer):
|
|
30
|
+
return self.model.encode([text], device=self.device).tolist()[0]
|
|
31
|
+
else:
|
|
32
|
+
return self.embed_documents([text])[0]
|
|
23
33
|
|
|
24
34
|
def mean_pool(self, last_hidden, mask):
|
|
25
35
|
# Shape: (batch, seq_len, dim), mask: (batch, seq_len, 1)
|
|
@@ -34,8 +44,10 @@ class BaseEmbeddings:
|
|
|
34
44
|
|
|
35
45
|
|
|
36
46
|
class MedGemmaEmbeddings(BaseEmbeddings):
|
|
37
|
-
def __init__(self, model_name="google/medgemma-4b-it", device=None):
|
|
38
|
-
super().__init__(
|
|
47
|
+
def __init__(self, model_name="google/medgemma-4b-it", model=None, device=None):
|
|
48
|
+
super().__init__(
|
|
49
|
+
model_name=model_name, device=device, model=model, trust_remote_code=True
|
|
50
|
+
)
|
|
39
51
|
|
|
40
52
|
def embed_documents(self, texts):
|
|
41
53
|
inputs = self.tokenizer(
|
|
@@ -55,8 +67,10 @@ class MedGemmaEmbeddings(BaseEmbeddings):
|
|
|
55
67
|
|
|
56
68
|
|
|
57
69
|
class QwenEmbeddings(BaseEmbeddings):
|
|
58
|
-
def __init__(self, model_name="Qwen/Qwen3-Embedding-0.6B", device=None):
|
|
59
|
-
super().__init__(
|
|
70
|
+
def __init__(self, model_name="Qwen/Qwen3-Embedding-0.6B", model=None, device=None):
|
|
71
|
+
super().__init__(
|
|
72
|
+
model_name=model_name, device=device, model=model, trust_remote_code=True
|
|
73
|
+
)
|
|
60
74
|
|
|
61
75
|
def embed_documents(self, texts):
|
|
62
76
|
inputs = self.tokenizer(
|
|
@@ -81,10 +95,14 @@ class QwenEmbeddings(BaseEmbeddings):
|
|
|
81
95
|
|
|
82
96
|
|
|
83
97
|
class BGEEmbeddings(BaseEmbeddings):
|
|
84
|
-
def __init__(self, model_name="BAAI/bge-large-en-v1.5", device=None):
|
|
85
|
-
super().__init__(model_name, device=device)
|
|
98
|
+
def __init__(self, model_name="BAAI/bge-large-en-v1.5", model=None, device=None):
|
|
99
|
+
super().__init__(model_name, device=device, model=model)
|
|
86
100
|
|
|
87
101
|
def embed_documents(self, texts):
|
|
102
|
+
# Special‑case: if we were given a SentenceTransformer, just use its `encode`.
|
|
103
|
+
if isinstance(self.model, SentenceTransformer):
|
|
104
|
+
return self.model.encode(texts, device=self.device).tolist()
|
|
105
|
+
|
|
88
106
|
inputs = self.tokenizer(
|
|
89
107
|
texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
|
|
90
108
|
).to(self.device)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai_harmonization
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: AI-backed harmonization framework
|
|
5
5
|
Requires-Python: <4.0.0,>=3.12
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -13,7 +13,6 @@ Requires-Dist: pydantic>=2.11.7
|
|
|
13
13
|
Requires-Dist: sssom>=0.4.16
|
|
14
14
|
Requires-Dist: sssom-schema>=1.0.0
|
|
15
15
|
Requires-Dist: tqdm>=4.67.1
|
|
16
|
-
Requires-Dist: transformers>=4.57.1
|
|
17
16
|
Requires-Dist: itables>=2.4.4
|
|
18
17
|
Requires-Dist: numpy>=2.0.2
|
|
19
18
|
Requires-Dist: pandas>=2.2.2
|
|
@@ -22,6 +21,9 @@ Requires-Dist: ipython>=7.34.0
|
|
|
22
21
|
Requires-Dist: torch>=2.8.0
|
|
23
22
|
Requires-Dist: matplotlib>=3.10.0
|
|
24
23
|
Requires-Dist: chromadb>=1.0.0
|
|
24
|
+
Requires-Dist: sentence-transformers>=5.1.2
|
|
25
|
+
Requires-Dist: datasets>=4.4.1
|
|
26
|
+
Requires-Dist: accelerate>=1.11.0
|
|
25
27
|
Dynamic: license-file
|
|
26
28
|
|
|
27
29
|
# AI Harmonization
|
|
@@ -6,7 +6,6 @@ pydantic>=2.11.7
|
|
|
6
6
|
sssom>=0.4.16
|
|
7
7
|
sssom-schema>=1.0.0
|
|
8
8
|
tqdm>=4.67.1
|
|
9
|
-
transformers>=4.57.1
|
|
10
9
|
itables>=2.4.4
|
|
11
10
|
numpy>=2.0.2
|
|
12
11
|
pandas>=2.2.2
|
|
@@ -15,3 +14,6 @@ ipython>=7.34.0
|
|
|
15
14
|
torch>=2.8.0
|
|
16
15
|
matplotlib>=3.10.0
|
|
17
16
|
chromadb>=1.0.0
|
|
17
|
+
sentence-transformers>=5.1.2
|
|
18
|
+
datasets>=4.4.1
|
|
19
|
+
accelerate>=1.11.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ai_harmonization"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "AI-backed harmonization framework"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12,<4.0.0"
|
|
@@ -13,7 +13,6 @@ dependencies = [
|
|
|
13
13
|
"sssom>=0.4.16",
|
|
14
14
|
"sssom-schema>=1.0.0",
|
|
15
15
|
"tqdm>=4.67.1",
|
|
16
|
-
"transformers>=4.57.1",
|
|
17
16
|
"itables>=2.4.4",
|
|
18
17
|
# We want this to work in a Google Colab environment.
|
|
19
18
|
# In such environments, Google's pre-installs a ton of Google
|
|
@@ -36,6 +35,9 @@ dependencies = [
|
|
|
36
35
|
# Similar to the above, leaving >= on an older version to
|
|
37
36
|
# automatically get the latest version possible during dep resolution
|
|
38
37
|
"chromadb>=1.0.0",
|
|
38
|
+
"sentence-transformers>=5.1.2",
|
|
39
|
+
"datasets>=4.4.1",
|
|
40
|
+
"accelerate>=1.11.0",
|
|
39
41
|
]
|
|
40
42
|
|
|
41
43
|
[tool.pytest.ini_options]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/base.py
RENAMED
|
File without changes
|
|
File without changes
|
{ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_benchmark.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|