ddi-fw 0.0.127__py3-none-any.whl → 0.0.129__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddi_fw/langchain/storage.py +6 -5
- {ddi_fw-0.0.127.dist-info → ddi_fw-0.0.129.dist-info}/METADATA +1 -1
- {ddi_fw-0.0.127.dist-info → ddi_fw-0.0.129.dist-info}/RECORD +5 -5
- {ddi_fw-0.0.127.dist-info → ddi_fw-0.0.129.dist-info}/WHEEL +0 -0
- {ddi_fw-0.0.127.dist-info → ddi_fw-0.0.129.dist-info}/top_level.txt +0 -0
ddi_fw/langchain/storage.py
CHANGED
|
@@ -137,7 +137,7 @@ class DataFrameToVectorDB:
|
|
|
137
137
|
total = 0
|
|
138
138
|
partial_dfs = split_dataframe(col_df, min_size=partial_df_size)
|
|
139
139
|
for partial_df in partial_dfs:
|
|
140
|
-
import torch
|
|
140
|
+
# import torch
|
|
141
141
|
|
|
142
142
|
documents = []
|
|
143
143
|
loader = DataFrameLoader(
|
|
@@ -148,11 +148,12 @@ class DataFrameToVectorDB:
|
|
|
148
148
|
split_docs_chunked = self.__split_list(
|
|
149
149
|
documents, self.batch_size)
|
|
150
150
|
for split_docs_chunk in split_docs_chunked:
|
|
151
|
-
|
|
151
|
+
print("entered chunks")
|
|
152
|
+
self.vectordb.add_documents(split_docs_chunk)
|
|
152
153
|
self.vectordb.persist()
|
|
153
154
|
total += len(partial_df)
|
|
154
155
|
print(f"{page_content_column}: {total}/{len(col_df)}")
|
|
155
|
-
torch.cuda.empty_cache()
|
|
156
|
+
# torch.cuda.empty_cache()
|
|
156
157
|
# time.sleep(30) # The GPU will not be used during this period
|
|
157
158
|
|
|
158
159
|
# split_docs_chunked = self.__split_list(
|
|
@@ -173,8 +174,8 @@ class DataFrameToVectorDB:
|
|
|
173
174
|
split_docs_chunked = self.__split_list(
|
|
174
175
|
documents, self.batch_size)
|
|
175
176
|
for split_docs_chunk in split_docs_chunked:
|
|
176
|
-
import torch
|
|
177
|
-
torch.cuda.empty_cache()
|
|
177
|
+
# import torch
|
|
178
|
+
# torch.cuda.empty_cache()
|
|
178
179
|
self.vectordb.add_documents(split_docs_chunk)
|
|
179
180
|
self.vectordb.persist()
|
|
180
181
|
print(f"{page_content_column}, size:{len(split_docs_chunk)}")
|
|
@@ -58,7 +58,7 @@ ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHr
|
|
|
58
58
|
ddi_fw/langchain/__init__.py,sha256=zS0CQrakWEP19biSRewFJGcBT8WBZq4899HrEKiMqUY,269
|
|
59
59
|
ddi_fw/langchain/embeddings.py,sha256=lU64a5AZ62jP8U3hTSwK0kXt7gThbwPACLfJMZ1baPA,7538
|
|
60
60
|
ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
|
|
61
|
-
ddi_fw/langchain/storage.py,sha256=
|
|
61
|
+
ddi_fw/langchain/storage.py,sha256=wLcJiVX_SsWDgMbOxQ3uUuzhn1cNUgGMDsBLYb2_K3c,11163
|
|
62
62
|
ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
|
|
63
63
|
ddi_fw/ml/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
|
|
64
64
|
ddi_fw/ml/ml_helper.py,sha256=fySjIAFzkeEOvaLJhDwtCOgRhgYQ7H106eqaP16GhDY,4489
|
|
@@ -91,7 +91,7 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
|
|
|
91
91
|
ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
|
|
92
92
|
ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
|
|
93
93
|
ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
|
|
94
|
-
ddi_fw-0.0.
|
|
95
|
-
ddi_fw-0.0.
|
|
96
|
-
ddi_fw-0.0.
|
|
97
|
-
ddi_fw-0.0.
|
|
94
|
+
ddi_fw-0.0.129.dist-info/METADATA,sha256=bs254f9Vihw4Sp8zV8ps-PU547T266MgfsNtLW3jPE0,1965
|
|
95
|
+
ddi_fw-0.0.129.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
96
|
+
ddi_fw-0.0.129.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
|
|
97
|
+
ddi_fw-0.0.129.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|