ddi-fw 0.0.66__py3-none-any.whl → 0.0.67__py3-none-any.whl

ddi_fw/datasets/__init__.py

@@ -2,7 +2,6 @@ from .core import BaseDataset
 from .ddi_mdl.base import DDIMDLDataset 
 from .mdf_sa_ddi.base import MDFSADDIDataset
 from .embedding_generator import create_embeddings
-from .embedding_generator_new import PoolingStrategy,SumPoolingStrategy,MeanPoolingStrategy,SentenceTransformerDecorator,PretrainedEmbeddings,SBertEmbeddings
 from .idf_helper import IDF
 from .feature_vector_generation import SimilarityMatrixGenerator, VectorGenerator
 

ddi_fw/datasets/core.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 import numpy as np
 import pandas as pd
 import pathlib
-from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
+from ddi_fw.langchain.embeddings import PoolingStrategy
 from ddi_fw.datasets.idf_helper import IDF
 
 from ddi_fw.utils.zip_helper import ZipHelper

ddi_fw/datasets/ddi_mdl/base.py

@@ -2,7 +2,7 @@ import pathlib
 
 import pandas as pd
 
-from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
+from ddi_fw.langchain.embeddings import PoolingStrategy
 from .. import BaseDataset
 from ..db_utils import create_connection
 

ddi_fw/experiments/pipeline.py

@@ -15,7 +15,7 @@ import chromadb
 from collections import defaultdict
 from langchain_community.vectorstores import Chroma
 from ddi_fw.ner.ner import CTakesNER
-from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
+from ddi_fw.langchain.embeddings import PoolingStrategy
 
 from ddi_fw.datasets import BaseDataset, DDIMDLDataset
 

ddi_fw/experiments/pipeline_builder_pattern.py

@@ -15,7 +15,7 @@ import chromadb
 from collections import defaultdict
 from langchain_community.vectorstores import Chroma
 from ddi_fw.ner.ner import CTakesNER
-from ddi_fw.datasets.embedding_generator_new import PoolingStrategy
+from ddi_fw.langchain.embeddings import PoolingStrategy
 
 from ddi_fw.datasets import BaseDataset, DDIMDLDataset
 

ddi_fw/langchain/__init__.py

@@ -0,0 +1,3 @@
+from ..langchain.embeddings import PoolingStrategy,SumPoolingStrategy,MeanPoolingStrategy,SentenceTransformerDecorator,PretrainedEmbeddings,SBertEmbeddings
+from .sentence_splitter import SentenceSplitter
+from .storage import DataFrameToVectorDB

ddi_fw/{datasets/embedding_generator_new.py → langchain/embeddings.py}

@@ -58,7 +58,7 @@ class SumPoolingStrategy(PoolingStrategy):
 
 class SentenceTransformerDecorator(Embeddings):
     def __init__(self, model_name="all-MiniLM-L6-v2", **kwargs: Any):
-        self.embeddings = SentenceTransformerEmbeddings(model_name=model_name)
+        self.embeddings = SentenceTransformerEmbeddings(model_name=model_name, **kwargs)
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
         return self.embeddings.embed_documents(texts)
@@ -67,7 +67,7 @@ class SentenceTransformerDecorator(Embeddings):
         return self.embeddings.embed_query(text)
 
 
-class PretrainedEmbeddings( Embeddings):
+class PretrainedEmbeddings(Embeddings):
     def __init__(self, model_name):
         self.mmodel_name = model_name
         self.model = AutoModel.from_pretrained(model_name)

ddi_fw/langchain/sentence_splitter.py

@@ -0,0 +1,10 @@
+from typing import List
+import nltk
+from nltk import sent_tokenize
+from langchain_text_splitters.base import TextSplitter
+
+nltk.download('punkt')
+
+class SentenceSplitter(TextSplitter):
+    def split_text(self, text: str) -> List[str]:
+        return sent_tokenize(text)

ddi_fw/langchain/storage.py

@@ -0,0 +1,69 @@
+from langchain.vectorstores import Chroma
+# from langchain_community.vectorstores import Chroma
+from langchain_community.vectorstores.utils import filter_complex_metadata
+from langchain_core.embeddings import Embeddings
+
+
+from langchain.docstore.document import Document
+
+from langchain.document_loaders import DataFrameLoader
+
+from langchain.text_splitter import TextSplitter, RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
+from sentence_splitter import SentenceSplitter
+
+from langchain_community.document_loaders.hugging_face_dataset import HuggingFaceDatasetLoader
+
+
+class DataFrameToVectorDB:
+    def __init__(self,
+                 collection_name,
+                 persist_directory,
+                 embeddings: Embeddings,
+                 text_splitter: TextSplitter, 
+                 chunk_size=1000, 
+                 chunk_overlap=20):
+        self.collection_name = collection_name
+        self.persist_directory = persist_directory
+        self.embeddings = embeddings
+        self.text_splitter = text_splitter
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.vectordb = Chroma(collection_name=collection_name,
+                               persist_directory=persist_directory,
+                               embedding_function=embeddings)
+
+    def __split_docs(self, documents):
+        docs = self.text_splitter.split_documents(documents)
+        return docs
+
+    def __split_list(self, input_list, chunk_size):
+        for i in range(0, len(input_list), chunk_size):
+            yield input_list[i:i + chunk_size]
+
+    def store(self, df, columns, page_content_columns, max_batch_size=1000):
+        for page_content_column in page_content_columns:
+            copy_columns = columns.copy()
+            copy_columns.append(page_content_column)
+            col_df = df[copy_columns].copy()
+            col_df.dropna(subset=[page_content_column], inplace=True)
+            col_df['type'] = page_content_column  # Set the type column
+            documents = []
+
+            loader = DataFrameLoader(data_frame=col_df, page_content_column=page_content_column)
+            loaded_docs = loader.load()
+            documents.extend(self.__split_docs(loaded_docs))
+
+            split_docs_chunked = self.__split_list(documents, max_batch_size)
+
+            for split_docs_chunk in split_docs_chunked:
+                # vectordb = Chroma.from_documents(
+                #     collection_name=collection_name,
+                #     documents=split_docs_chunk,
+                #     embedding=embeddings,
+                #     persist_directory=persist_directory,
+                # )
+                self.vectordb.add_documents(split_docs_chunk)
+                self.vectordb.persist()
+
+
+ 

{ddi_fw-0.0.66.dist-info → ddi_fw-0.0.67.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.66
+Version: 0.0.67
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.66.dist-info → ddi_fw-0.0.67.dist-info}/RECORD

@@ -1,12 +1,11 @@
-ddi_fw/datasets/__init__.py,sha256=CqDrx7Ov83pXRh-n0ylembBmzhlW_yFWiheBcISrKdg,510
-ddi_fw/datasets/core.py,sha256=ffza6yX3zvZV8Lp7but5f49J0837gZQKCSQ3iMBT6BE,19033
+ddi_fw/datasets/__init__.py,sha256=HSwQrqnzrEjIG4gif41pwJ_cST3t2XHGDxqFyuEBRwo,351
+ddi_fw/datasets/core.py,sha256=lGVP2P8CIeSEG5fH230XV8bLoycblJxBQKYbdMSBITM,19021
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/embedding_generator.py,sha256=Jqrlv88RCu0Lg812KsA12X0cSaZuxbckJ4LNRKNy_qw,2173
-ddi_fw/datasets/embedding_generator_new.py,sha256=GOE-Io6-DBwiUJSkgmxw9ZM1exCYYVu9KyP2dH3gf1o,7506
 ddi_fw/datasets/feature_vector_generation.py,sha256=EImavcALxkIB0YG_smOzagMNzuWMbK9SaWSKwARx_qU,3254
 ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=7MMAMJtJqqTXmYiI4Hf7e4kO2xoOynPocHazJrE9Y_w,4261
+ddi_fw/datasets/ddi_mdl/base.py,sha256=ZW8uJIvEizK2x_VkoyhNYcKh3ki3kQRsKxl8d2_hVYQ,4249
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
 ddi_fw/datasets/ddi_mdl/indexes/test_indexes.txt,sha256=XVlDqYATckrQwNSXqMSKVBqyoN_Hg8SK6CL-XMdLADY,102176
@@ -59,11 +58,15 @@ ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHr
 ddi_fw/experiments/__init__.py,sha256=5L2xSolpFycNnflqOMdvJSiqRB16ExA5bbVGORKFX04,195
 ddi_fw/experiments/custom_torch_model.py,sha256=iQ_R_EApzD2JCcASN8cie6D21oh7VCxaOQ45_dkiGwc,2576
 ddi_fw/experiments/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
-ddi_fw/experiments/pipeline.py,sha256=dI_yxBt73hGFMxFIVhCRSjrefaGbILfQXCHr9VQ8tI8,5693
-ddi_fw/experiments/pipeline_builder_pattern.py,sha256=q1PNEQFoO5U3UidEoGB8rgLA7KXr4FsJTXEug5c5UJg,5466
+ddi_fw/experiments/pipeline.py,sha256=N07EBv2IGa9oD0A1XxvUktDjGHi0SFmt3QqupF2rs3k,5681
+ddi_fw/experiments/pipeline_builder_pattern.py,sha256=w6x7ietk4vONCAvUfssPycaRUQIYUJsbCNNj3BTASBI,5454
 ddi_fw/experiments/pipeline_ner.py,sha256=unxEJCYrG6wEZjLmqvGdLRTMOBwELbGKkdygSpAR3b8,5043
 ddi_fw/experiments/tensorflow_helper.py,sha256=xUnbntWyc2Wm4TvmVFAnpwLHg-o13oM26GUHom6d5m0,11776
 ddi_fw/experiments/test.py,sha256=z1TfBpK75zGKpp2ZU8f6APjZlgBFthaCBN61YB9ma4o,2049
+ddi_fw/langchain/__init__.py,sha256=8dBPZivc01WWaCH8sZ_UV8-XPyo74e9Qy6-fYgAiNLE,248
+ddi_fw/langchain/embeddings.py,sha256=8J_SfO9pyET2W-Ltzq0_r9EchFzBsYdUabiOMma42Us,7515
+ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
+ddi_fw/langchain/storage.py,sha256=xGCpqeRIPU_S2In2JcFW3ixnsdfZ7d3Ux0xJNbV4h6s,2833
 ddi_fw/ner/__init__.py,sha256=JwhGXrepomxPSsGsg2b_xPRC72AjvxOIn2CW5Mvscn0,26
 ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6859
 ddi_fw/ner/ner.py,sha256=BEs9AFljAxOQrC2BEP1raSzRoypcfELS5UTdl4bjTqw,15863
@@ -83,7 +86,7 @@ ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.66.dist-info/METADATA,sha256=r5ST3WA9Wzm2x7LQOBUpt6E8ltZjimmA-MWrV6n5ubE,1565
-ddi_fw-0.0.66.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-ddi_fw-0.0.66.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.66.dist-info/RECORD,,
+ddi_fw-0.0.67.dist-info/METADATA,sha256=UvAmId4nJachjVEKPzm1_CRK8N2NiamboJpdosGxzew,1565
+ddi_fw-0.0.67.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+ddi_fw-0.0.67.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.67.dist-info/RECORD,,