ai_harmonization 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/PKG-INFO +4 -2
  2. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/embeddings.py +28 -10
  3. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/PKG-INFO +4 -2
  4. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/requires.txt +3 -1
  5. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/pyproject.toml +4 -2
  6. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/LICENSE +0 -0
  7. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/README.md +0 -0
  8. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/__init__.py +0 -0
  9. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/cde.py +0 -0
  10. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/__init__.py +0 -0
  11. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/base.py +0 -0
  12. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_approaches/similarity_inmem.py +0 -0
  13. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/harmonization_benchmark.py +0 -0
  14. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/interactive.py +0 -0
  15. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/jsonl.py +0 -0
  16. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/main.py +0 -0
  17. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/simple_data_model.py +0 -0
  18. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization/utils.py +0 -0
  19. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/SOURCES.txt +0 -0
  20. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/dependency_links.txt +0 -0
  21. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/ai_harmonization.egg-info/top_level.txt +0 -0
  22. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/setup.cfg +0 -0
  23. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/tests/test_harmonization.py +0 -0
  24. {ai_harmonization-0.3.2 → ai_harmonization-0.4.0}/tests/test_sssom.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ai_harmonization
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: AI-backed harmonization framework
5
5
  Requires-Python: <4.0.0,>=3.12
6
6
  Description-Content-Type: text/markdown
@@ -13,7 +13,6 @@ Requires-Dist: pydantic>=2.11.7
13
13
  Requires-Dist: sssom>=0.4.16
14
14
  Requires-Dist: sssom-schema>=1.0.0
15
15
  Requires-Dist: tqdm>=4.67.1
16
- Requires-Dist: transformers>=4.57.1
17
16
  Requires-Dist: itables>=2.4.4
18
17
  Requires-Dist: numpy>=2.0.2
19
18
  Requires-Dist: pandas>=2.2.2
@@ -22,6 +21,9 @@ Requires-Dist: ipython>=7.34.0
22
21
  Requires-Dist: torch>=2.8.0
23
22
  Requires-Dist: matplotlib>=3.10.0
24
23
  Requires-Dist: chromadb>=1.0.0
24
+ Requires-Dist: sentence-transformers>=5.1.2
25
+ Requires-Dist: datasets>=4.4.1
26
+ Requires-Dist: accelerate>=1.11.0
25
27
  Dynamic: license-file
26
28
 
27
29
  # AI Harmonization
@@ -1,25 +1,35 @@
1
1
  import numpy as np
2
2
  import torch
3
3
  from transformers import AutoModel, AutoTokenizer
4
+ from sentence_transformers import SentenceTransformer
4
5
 
5
6
 
6
7
  class BaseEmbeddings:
7
- def __init__(self, model_name, device=None, trust_remote_code=False):
8
+ def __init__(self, model_name, model=None, device=None, trust_remote_code=False):
8
9
  # Load tokenizer/model, set device, set eval()
9
- self.tokenizer = AutoTokenizer.from_pretrained(
10
+ tokenizer = None
11
+ if model:
12
+ tokenizer = model.tokenizer
13
+
14
+ self.model_name = model_name
15
+
16
+ self.tokenizer = tokenizer or AutoTokenizer.from_pretrained(
10
17
  model_name, trust_remote_code=trust_remote_code
11
18
  )
12
19
  self.device = torch.device(
13
20
  device if device else ("cuda" if torch.cuda.is_available() else "cpu")
14
21
  )
15
- self.model = (
22
+ self.model = model or (
16
23
  AutoModel.from_pretrained(model_name, trust_remote_code=trust_remote_code)
17
24
  .to(self.device)
18
25
  .eval()
19
26
  )
20
27
 
21
28
  def embed_query(self, text):
22
- return self.embed_documents([text])[0]
29
+ if isinstance(self.model, SentenceTransformer):
30
+ return self.model.encode([text], device=self.device).tolist()[0]
31
+ else:
32
+ return self.embed_documents([text])[0]
23
33
 
24
34
  def mean_pool(self, last_hidden, mask):
25
35
  # Shape: (batch, seq_len, dim), mask: (batch, seq_len, 1)
@@ -34,8 +44,10 @@ class BaseEmbeddings:
34
44
 
35
45
 
36
46
  class MedGemmaEmbeddings(BaseEmbeddings):
37
- def __init__(self, model_name="google/medgemma-4b-it", device=None):
38
- super().__init__(model_name, device, trust_remote_code=True)
47
+ def __init__(self, model_name="google/medgemma-4b-it", model=None, device=None):
48
+ super().__init__(
49
+ model_name=model_name, device=device, model=model, trust_remote_code=True
50
+ )
39
51
 
40
52
  def embed_documents(self, texts):
41
53
  inputs = self.tokenizer(
@@ -55,8 +67,10 @@ class MedGemmaEmbeddings(BaseEmbeddings):
55
67
 
56
68
 
57
69
  class QwenEmbeddings(BaseEmbeddings):
58
- def __init__(self, model_name="Qwen/Qwen3-Embedding-0.6B", device=None):
59
- super().__init__(model_name, device, trust_remote_code=True)
70
+ def __init__(self, model_name="Qwen/Qwen3-Embedding-0.6B", model=None, device=None):
71
+ super().__init__(
72
+ model_name=model_name, device=device, model=model, trust_remote_code=True
73
+ )
60
74
 
61
75
  def embed_documents(self, texts):
62
76
  inputs = self.tokenizer(
@@ -81,10 +95,14 @@ class QwenEmbeddings(BaseEmbeddings):
81
95
 
82
96
 
83
97
  class BGEEmbeddings(BaseEmbeddings):
84
- def __init__(self, model_name="BAAI/bge-large-en-v1.5", device=None):
85
- super().__init__(model_name, device=device)
98
+ def __init__(self, model_name="BAAI/bge-large-en-v1.5", model=None, device=None):
99
+ super().__init__(model_name, device=device, model=model)
86
100
 
87
101
  def embed_documents(self, texts):
102
+ # Special‑case: if we were given a SentenceTransformer, just use its `encode`.
103
+ if isinstance(self.model, SentenceTransformer):
104
+ return self.model.encode(texts, device=self.device).tolist()
105
+
88
106
  inputs = self.tokenizer(
89
107
  texts, padding=True, truncation=True, max_length=512, return_tensors="pt"
90
108
  ).to(self.device)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ai_harmonization
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: AI-backed harmonization framework
5
5
  Requires-Python: <4.0.0,>=3.12
6
6
  Description-Content-Type: text/markdown
@@ -13,7 +13,6 @@ Requires-Dist: pydantic>=2.11.7
13
13
  Requires-Dist: sssom>=0.4.16
14
14
  Requires-Dist: sssom-schema>=1.0.0
15
15
  Requires-Dist: tqdm>=4.67.1
16
- Requires-Dist: transformers>=4.57.1
17
16
  Requires-Dist: itables>=2.4.4
18
17
  Requires-Dist: numpy>=2.0.2
19
18
  Requires-Dist: pandas>=2.2.2
@@ -22,6 +21,9 @@ Requires-Dist: ipython>=7.34.0
22
21
  Requires-Dist: torch>=2.8.0
23
22
  Requires-Dist: matplotlib>=3.10.0
24
23
  Requires-Dist: chromadb>=1.0.0
24
+ Requires-Dist: sentence-transformers>=5.1.2
25
+ Requires-Dist: datasets>=4.4.1
26
+ Requires-Dist: accelerate>=1.11.0
25
27
  Dynamic: license-file
26
28
 
27
29
  # AI Harmonization
@@ -6,7 +6,6 @@ pydantic>=2.11.7
6
6
  sssom>=0.4.16
7
7
  sssom-schema>=1.0.0
8
8
  tqdm>=4.67.1
9
- transformers>=4.57.1
10
9
  itables>=2.4.4
11
10
  numpy>=2.0.2
12
11
  pandas>=2.2.2
@@ -15,3 +14,6 @@ ipython>=7.34.0
15
14
  torch>=2.8.0
16
15
  matplotlib>=3.10.0
17
16
  chromadb>=1.0.0
17
+ sentence-transformers>=5.1.2
18
+ datasets>=4.4.1
19
+ accelerate>=1.11.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ai_harmonization"
3
- version = "0.3.2"
3
+ version = "0.4.0"
4
4
  description = "AI-backed harmonization framework"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12,<4.0.0"
@@ -13,7 +13,6 @@ dependencies = [
13
13
  "sssom>=0.4.16",
14
14
  "sssom-schema>=1.0.0",
15
15
  "tqdm>=4.67.1",
16
- "transformers>=4.57.1",
17
16
  "itables>=2.4.4",
18
17
  # We want this to work in a Google Colab environment.
19
18
  # In such environments, Google's pre-installs a ton of Google
@@ -36,6 +35,9 @@ dependencies = [
36
35
  # Similar to the above, leaving >= on an older version to
37
36
  # automatically get the latest version possible during dep resolution
38
37
  "chromadb>=1.0.0",
38
+ "sentence-transformers>=5.1.2",
39
+ "datasets>=4.4.1",
40
+ "accelerate>=1.11.0",
39
41
  ]
40
42
 
41
43
  [tool.pytest.ini_options]