ddi-fw 0.0.102__py3-none-any.whl → 0.0.104__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddi_fw/langchain/embeddings.py +1 -0
- ddi_fw/langchain/storage.py +9 -6
- {ddi_fw-0.0.102.dist-info → ddi_fw-0.0.104.dist-info}/METADATA +1 -1
- {ddi_fw-0.0.102.dist-info → ddi_fw-0.0.104.dist-info}/RECORD +6 -6
- {ddi_fw-0.0.102.dist-info → ddi_fw-0.0.104.dist-info}/WHEEL +0 -0
- {ddi_fw-0.0.102.dist-info → ddi_fw-0.0.104.dist-info}/top_level.txt +0 -0
ddi_fw/langchain/embeddings.py
CHANGED
ddi_fw/langchain/storage.py
CHANGED
@@ -12,6 +12,7 @@ from langchain.document_loaders import DataFrameLoader
|
|
12
12
|
from langchain.text_splitter import TextSplitter
|
13
13
|
|
14
14
|
# from langchain_community.document_loaders.hugging_face_dataset import HuggingFaceDatasetLoader
|
15
|
+
from ddi_fw.langchain.embeddings import SBertEmbeddings
|
15
16
|
from ddi_fw.utils import get_import
|
16
17
|
|
17
18
|
|
@@ -134,8 +135,10 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
|
|
134
135
|
|
135
136
|
# Assuming the classes for the embeddings and splitters are available
|
136
137
|
try:
|
138
|
+
model_kwargs = collection_config['model_kwargs']
|
139
|
+
SBertEmbeddings(model_name="a", model_config={})
|
137
140
|
model = get_import(embedding_model_type)(
|
138
|
-
model_name=name, model_kwargs
|
141
|
+
model_name=name, **model_kwargs)
|
139
142
|
except:
|
140
143
|
# print(f"Unknown embedding model: {embedding_model_type}")
|
141
144
|
raise Exception(f"Unknown embedding model: {embedding_model_type}")
|
@@ -144,12 +147,12 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
|
|
144
147
|
text_splitters_suffixes = []
|
145
148
|
for text_splitter_type in text_splitters_types:
|
146
149
|
try:
|
147
|
-
type_of_text_splitter = get_import(text_splitter_type)
|
148
|
-
|
149
|
-
suffix =
|
150
|
-
if
|
150
|
+
type_of_text_splitter = get_import(text_splitter_type.get("type"))
|
151
|
+
kwargs = text_splitter_type.get("params")
|
152
|
+
suffix = text_splitter_type.get("suffix")
|
153
|
+
if kwargs:
|
151
154
|
text_splitter = type_of_text_splitter(
|
152
|
-
**
|
155
|
+
**kwargs)
|
153
156
|
else:
|
154
157
|
text_splitter = type_of_text_splitter()
|
155
158
|
text_splitters.append(text_splitter)
|
@@ -56,9 +56,9 @@ ddi_fw/drugbank/drugbank_processor.py,sha256=vmkt68n9nFLevufgGyXhOSDtTo4G1XzwT9P
|
|
56
56
|
ddi_fw/drugbank/drugbank_processor_org.py,sha256=eO5Yset50P91qkic79RUXPoEuxRxQKFkKW0l4G29Mas,13322
|
57
57
|
ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHrBXsI8,4600
|
58
58
|
ddi_fw/langchain/__init__.py,sha256=zS0CQrakWEP19biSRewFJGcBT8WBZq4899HrEKiMqUY,269
|
59
|
-
ddi_fw/langchain/embeddings.py,sha256=
|
59
|
+
ddi_fw/langchain/embeddings.py,sha256=lU64a5AZ62jP8U3hTSwK0kXt7gThbwPACLfJMZ1baPA,7538
|
60
60
|
ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
|
61
|
-
ddi_fw/langchain/storage.py,sha256=
|
61
|
+
ddi_fw/langchain/storage.py,sha256=Swxh5mSFAb6q1R2ZHgYy89OC3iSKtVoKjRNJERYoJCA,7561
|
62
62
|
ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
|
63
63
|
ddi_fw/ml/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
|
64
64
|
ddi_fw/ml/ml_helper.py,sha256=fySjIAFzkeEOvaLJhDwtCOgRhgYQ7H106eqaP16GhDY,4489
|
@@ -91,7 +91,7 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
|
|
91
91
|
ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
|
92
92
|
ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
|
93
93
|
ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
|
94
|
-
ddi_fw-0.0.
|
95
|
-
ddi_fw-0.0.
|
96
|
-
ddi_fw-0.0.
|
97
|
-
ddi_fw-0.0.
|
94
|
+
ddi_fw-0.0.104.dist-info/METADATA,sha256=yWmrr5MurOmnjfKPsZtSpY7hyO0ejmuovv_Om09VJiM,1967
|
95
|
+
ddi_fw-0.0.104.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
96
|
+
ddi_fw-0.0.104.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
|
97
|
+
ddi_fw-0.0.104.dist-info/RECORD,,
|
File without changes
|
File without changes
|