ddi-fw 0.0.126__py3-none-any.whl → 0.0.127__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. ddi_fw/langchain/storage.py +6 -4
  2. {ddi_fw-0.0.126.dist-info → ddi_fw-0.0.127.dist-info}/METADATA +1 -1
  3. {ddi_fw-0.0.126.dist-info → ddi_fw-0.0.127.dist-info}/RECORD +5 -5
  4. {ddi_fw-0.0.126.dist-info → ddi_fw-0.0.127.dist-info}/WHEEL +0 -0
  5. {ddi_fw-0.0.126.dist-info → ddi_fw-0.0.127.dist-info}/top_level.txt +0 -0
ddi_fw/langchain/storage.py CHANGED
@@ -144,11 +144,13 @@ class DataFrameToVectorDB:
144
144
  data_frame=partial_df, page_content_column=page_content_column)
145
145
  loaded_docs = loader.load()
146
146
  # print(loaded_docs)
147
- # documents.extend(self.__split_docs(loaded_docs))
147
+ documents.extend(self.__split_docs(loaded_docs))
148
+ split_docs_chunked = self.__split_list(
149
+ documents, self.batch_size)
150
+ for split_docs_chunk in split_docs_chunked:
151
+ self.vectordb.add_documents(loaded_docs)
152
+ self.vectordb.persist()
148
153
  total += len(partial_df)
149
-
150
- self.vectordb.add_documents(loaded_docs)
151
- self.vectordb.persist()
152
154
  print(f"{page_content_column}: {total}/{len(col_df)}")
153
155
  torch.cuda.empty_cache()
154
156
  # time.sleep(30) # The GPU will not be used during this period
{ddi_fw-0.0.126.dist-info → ddi_fw-0.0.127.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ddi_fw
3
- Version: 0.0.126
3
+ Version: 0.0.127
4
4
  Summary: Do not use :)
5
5
  Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
6
6
  Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
{ddi_fw-0.0.126.dist-info → ddi_fw-0.0.127.dist-info}/RECORD RENAMED
@@ -58,7 +58,7 @@ ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHr
58
58
  ddi_fw/langchain/__init__.py,sha256=zS0CQrakWEP19biSRewFJGcBT8WBZq4899HrEKiMqUY,269
59
59
  ddi_fw/langchain/embeddings.py,sha256=lU64a5AZ62jP8U3hTSwK0kXt7gThbwPACLfJMZ1baPA,7538
60
60
  ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
61
- ddi_fw/langchain/storage.py,sha256=INiAYeP714iyTLUj-aiyl7GM56qwdhpw6eVO81qbHFU,10918
61
+ ddi_fw/langchain/storage.py,sha256=sWeFJgCLKtMFsBev1cJ4SKy_aoG5cTx9IadY-7S99es,11101
62
62
  ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
63
63
  ddi_fw/ml/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
64
64
  ddi_fw/ml/ml_helper.py,sha256=fySjIAFzkeEOvaLJhDwtCOgRhgYQ7H106eqaP16GhDY,4489
@@ -91,7 +91,7 @@ ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5
91
91
  ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
92
92
  ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
93
93
  ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
94
- ddi_fw-0.0.126.dist-info/METADATA,sha256=gGpAp81CDpjQU_v_8zlSYN5oxe3a6DBzbXEZfFM4iOY,1965
95
- ddi_fw-0.0.126.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
96
- ddi_fw-0.0.126.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
97
- ddi_fw-0.0.126.dist-info/RECORD,,
94
+ ddi_fw-0.0.127.dist-info/METADATA,sha256=HwCn3w63ygXynQWJopR8pf2ltuqBimqZB-Sk-zxTmk8,1965
95
+ ddi_fw-0.0.127.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
96
+ ddi_fw-0.0.127.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
97
+ ddi_fw-0.0.127.dist-info/RECORD,,