ddi-fw 0.0.256__py3-none-any.whl → 0.0.258__py3-none-any.whl

ddi_fw/langchain/chroma_storage.py

@@ -86,7 +86,7 @@ def split_dataframe_indices(df, min_size=512):
 
 class ChromaVectorStoreManager(BaseVectorStoreManager):
     collection_name: str = Field(default="default")
-    persist_directory: str = Field(default="./chroma_db")
+    persist_directory: str = Field(default=".embeddings/chroma_db")
     text_splitter: Optional[TextSplitter] = None
     batch_size: int = Field(default=1024)
     vector_store: Optional[Chroma] | None = None
@@ -152,7 +152,9 @@ class ChromaVectorStoreManager(BaseVectorStoreManager):
     #                 self.vector_store.persist()
     #                 print(f"{page_content_column}, size:{len(split_docs_chunk)}")
 
-
+    @staticmethod
+    def get_persist_dir(base_dir, suffix, config=None):
+        return f"{base_dir}"
 
     def generate_vector_store(self, docs: List[Document]):
         self.vector_store = Chroma(

ddi_fw/langchain/faiss_storage.py

@@ -35,6 +35,11 @@ class BaseVectorStoreManager(BaseModel):
     
     def as_dataframe(self, formatter_fn: Optional[Callable[[Document, np.ndarray], Dict[str, Any]]] = None) -> pd.DataFrame:
         raise NotImplementedError("This method should be implemented by subclasses.")
+    
+    @staticmethod
+    def get_persist_dir(base_dir, suffix, config=None):
+        raise NotImplementedError("Subclasses must implement get_persist_dir.")
+     
 
 class FaissVectorStoreManager(BaseVectorStoreManager):
     persist_directory: str = Field(default="./embeddings/faiss")
@@ -57,6 +62,10 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
     #     uuids = [str(uuid4()) for _ in range(len(docs))]
     #     self.vector_store.add_documents(documents=docs, ids=uuids)
     
+    @staticmethod
+    def get_persist_dir(base_dir, suffix, config=None):
+        return f"{base_dir}/{suffix}"
+    
     def initialize_embedding_dict(self, **kwargs):
         """
         Initializes a dictionary where keys are types (e.g., 'description', 'indication'),
@@ -353,7 +362,7 @@ def load_configuration(config_file):
 #             # Optionally persist/save
 #             vector_store_manager.save(persist_dir)
 
-
+# persist_directory config'den alınsın
 def generate_embeddings(
     docs,
     vector_store_manager_type:Type[BaseVectorStoreManager],
@@ -421,7 +430,9 @@ def generate_embeddings(
         for text_splitter, suffix in zip(text_splitters, text_splitters_suffixes):
             print(f"{id}_{suffix}")
             # persist_dir = f'{persist_directory}/{id}/{suffix}'
-            persist_dir = f'{persist_directory}/{suffix}'
+            # persist_dir = f'{persist_directory}/{suffix}'
+            persist_dir = vector_store_manager_type.get_persist_dir(persist_directory, suffix, collection_config)
+
             # Prepare manager parameters
             manager_params = {
                 "collection_name": f"{id}_{suffix}",

{ddi_fw-0.0.256.dist-info → ddi_fw-0.0.258.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.256
+Version: 0.0.258
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.256.dist-info → ddi_fw-0.0.258.dist-info}/RECORD

@@ -4,9 +4,9 @@ ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
 ddi_fw/langchain/__init__.py,sha256=Kk2Yr7vemjy9MNB_ImAWET808zt1JkLsWqsgEXpVPJk,421
-ddi_fw/langchain/chroma_storage.py,sha256=7LSUhdiPdQHZvKC_NapOeVbHtS71iE5ABZVTrI0YQ-A,15520
+ddi_fw/langchain/chroma_storage.py,sha256=Xh0p2d8bI9U-NYkhuu7fLtsgbExP7phkqPE17PBYWd4,15634
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=LG2cf6upCEHBBF4Jixnq_diA_t3A0OQZBqPc9DwmILc,17825
+ddi_fw/langchain/faiss_storage.py,sha256=kFnvsRCE1Kdv77XgC2jLlYrAHZ9Qoz-ygs-IPB2Sg6g,18275
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -38,7 +38,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.256.dist-info/METADATA,sha256=4Gwtn1oiXo2cJ8osjOXRRu7JIQuP8i3PWjOsE7B56t0,2623
-ddi_fw-0.0.256.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.256.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.256.dist-info/RECORD,,
+ddi_fw-0.0.258.dist-info/METADATA,sha256=3r-JrOhNUwt3sUXiIj0Ua8a92c67V324tVHDxTBMM10,2623
+ddi_fw-0.0.258.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.258.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.258.dist-info/RECORD,,