ddi-fw 0.0.261__py3-none-any.whl → 0.0.262__py3-none-any.whl

ddi_fw/langchain/__init__.py

@@ -1,5 +1,5 @@
 from ..langchain.embeddings import PoolingStrategy,SumPoolingStrategy,MeanPoolingStrategy,SentenceTransformerDecorator,PretrainedEmbeddings,SBertEmbeddings
-from .sentence_splitter import SentenceSplitter
+from .sentence_splitter import SentenceSplitter,PassthroughTextSplitter
 # from .storage import DataFrameToVectorDB, generate_embeddings
 from .faiss_storage import BaseVectorStoreManager, FaissVectorStoreManager,generate_embeddings
 from .chroma_storage import ChromaVectorStoreManager

ddi_fw/langchain/chroma_storage.py

@@ -153,8 +153,8 @@ class ChromaVectorStoreManager(BaseVectorStoreManager):
     #                 print(f"{page_content_column}, size:{len(split_docs_chunk)}")
 
     @staticmethod
-    def get_persist_dir(base_dir, suffix, config=None):
-        return f"{base_dir}"
+    def get_persist_dir(base_dir,id, suffix, config=None):
+        return f"{base_dir}/chroma_db/{id}"
 
     def generate_vector_store(self, docs: List[Document]):
         self.vector_store = Chroma(

ddi_fw/langchain/faiss_storage.py

@@ -39,7 +39,7 @@ class BaseVectorStoreManager(BaseModel):
         raise NotImplementedError("This method should be implemented by subclasses.")
     
     @staticmethod
-    def get_persist_dir(base_dir, suffix, config=None):
+    def get_persist_dir(base_dir, id ,suffix, config=None):
         raise NotImplementedError("Subclasses must implement get_persist_dir.")
      
 
@@ -66,8 +66,8 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
     #     self.vector_store.add_documents(documents=docs, ids=uuids)
     
     @staticmethod
-    def get_persist_dir(base_dir, suffix, config=None):
-        return f"{base_dir}/{suffix}"
+    def get_persist_dir(base_dir,id, suffix, config=None):
+        return f"{base_dir}/faiss/{id}/{suffix}"
     
     def initialize_embedding_dict(self, **kwargs):
         """
@@ -441,7 +441,7 @@ def generate_embeddings(
             print(f"{id}_{suffix}")
             # persist_dir = f'{persist_directory}/{id}/{suffix}'
             # persist_dir = f'{persist_directory}/{suffix}'
-            persist_dir = vector_store_manager_type.get_persist_dir(persist_directory, suffix, collection_config)
+            persist_dir = vector_store_manager_type.get_persist_dir(persist_directory , id, suffix, collection_config)
 
             # Prepare manager parameters
             manager_params = {

ddi_fw/langchain/sentence_splitter.py

@@ -5,6 +5,13 @@ from langchain_text_splitters.base import TextSplitter
 
 nltk.download('punkt')
 
+''' A text splitter that splits text into sentences using NLTK's sentence tokenizer.'''
 class SentenceSplitter(TextSplitter):
     def split_text(self, text: str) -> List[str]:
         return sent_tokenize(text)
+
+
+''' A text splitter that does not split the text at all, returning the entire text as a single chunk.'''
+class PassthroughTextSplitter(TextSplitter):
+    def split_text(self, text: str) -> List[str]:
+        return [text]

{ddi_fw-0.0.261.dist-info → ddi_fw-0.0.262.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.261
+Version: 0.0.262
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.261.dist-info → ddi_fw-0.0.262.dist-info}/RECORD

@@ -3,11 +3,11 @@ ddi_fw/datasets/core.py,sha256=UnbCDoWXdxeiAb0e0anhDqXiVFGUi02VA9sKl6NVBZU,17409
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/langchain/__init__.py,sha256=Kk2Yr7vemjy9MNB_ImAWET808zt1JkLsWqsgEXpVPJk,421
-ddi_fw/langchain/chroma_storage.py,sha256=63_UojxGLbytgm4g2BZWdo2hvnWiVjrs4mZjxNxdkV8,15837
+ddi_fw/langchain/__init__.py,sha256=97Y4lYuxShWqx5hfDbzf8VyV0HrM76fDlNp5xXusKQU,445
+ddi_fw/langchain/chroma_storage.py,sha256=fOxoJoaqqyOKqtfUtlq2zJd-XY03rARTDvrPE_9nY2I,15855
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=1G_lJ4_pKGEp5SlKBYUZWxEnGHuq3JGmvKeDaEztX8w,18646
-ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
+ddi_fw/langchain/faiss_storage.py,sha256=M_i1WSb66u14pIDnQ9YXTw1XNgPzsy2vchxaSzc6vY4,18669
+ddi_fw/langchain/sentence_splitter.py,sha256=NCcDdDWDnwZTZDqarg-5gSbcDFoAM_sxcgH9ZCu97IA,597
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
 ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
@@ -38,7 +38,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.261.dist-info/METADATA,sha256=P0xexzVAJfggUMTjYeClVcqkap4mhjRzzLJCfDR86II,2623
-ddi_fw-0.0.261.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.261.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.261.dist-info/RECORD,,
+ddi_fw-0.0.262.dist-info/METADATA,sha256=HDaqqJBBz5EFNJ8ggBj4Si6Qib0mSlih4jqdeUEnU-Q,2623
+ddi_fw-0.0.262.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.262.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.262.dist-info/RECORD,,