ddi-fw 0.0.247__tar.gz → 0.0.249__tar.gz

{ddi_fw-0.0.247 → ddi_fw-0.0.249}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.247
+Version: 0.0.249
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.247 → ddi_fw-0.0.249}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.247"
+version = "0.0.249"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.247 → ddi_fw-0.0.249}/src/ddi_fw/langchain/faiss_storage.py

@@ -3,7 +3,7 @@ import pandas as pd
 from uuid import uuid4
 from langchain_community.vectorstores.faiss import FAISS
 from langchain_community.docstore.in_memory import InMemoryDocstore
-from typing import Callable, Optional, Dict, Any
+from typing import Callable, Optional, Dict, Any, Type
 from langchain_core.documents import Document
 import numpy as np  # optional, if you're using NumPy vectors
 from langchain_core.embeddings import Embeddings
@@ -11,6 +11,8 @@ from langchain_core.embeddings import Embeddings
 from pydantic import BaseModel, Field
 from langchain_core.embeddings import Embeddings
 from langchain_core.vectorstores import VectorStore
+from ddi_fw.utils import get_import
+from langchain.document_loaders import DataFrameLoader
 
 class BaseVectorStoreManager(BaseModel):
     embeddings: Optional[Embeddings] = None
@@ -54,7 +56,9 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
     #     uuids = [str(uuid4()) for _ in range(len(docs))]
     #     self.vector_store.add_documents(documents=docs, ids=uuids)
         
-    def initialize_embedding_dict(self):
+    def initialize_embedding_dict(self, **kwargs):
+        vector_db_persist_directory = kwargs.get("vector_db_persist_directory")
+        self.load(vector_db_persist_directory)
         df = self.as_dataframe(formatter_fn=custom_formatter )
         type_dict = (
             df.groupby('type')
@@ -240,3 +244,106 @@ def custom_formatter(document: Document, vector: np.ndarray) -> Dict[str, Any]:
         "type": document.metadata.get("type", None),
         "embedding": vector
     }
+
+def load_configuration(config_file):
+    """
+    Load the configuration from a JSON file.
+    """
+    import json
+    with open(config_file, 'r') as f:
+        config = json.load(f)
+    return config
+
+
+def generate_embeddings(
+    df,
+    vector_store_manager_type:Type[BaseVectorStoreManager], 
+    config_file,
+    new_model_names,
+    collections,
+    persist_directory="embeddings",
+):
+    """
+    Generate embeddings for collections based on a configuration file.
+
+    collections: List of collections that contain metadata for embedding generation.
+    config_file: Path to the configuration file containing model settings.
+    new_model_names: List of model names to generate embeddings for.
+    vector_store_manager_type: Class type of the vector store manager (e.g., FaissVectorStoreManager or ChromaVectorStoreManager)
+    """
+    if not collections and not config_file:
+        raise ValueError("Either 'collections' or 'config_file' must be provided.")
+    if collections and config_file:
+        raise ValueError("Only one of 'collections' or 'config_file' should be provided.")
+    
+    if not collections:
+        collections = load_configuration(config_file)
+
+    for collection_config in collections:
+        id = collection_config['id']
+        name = collection_config['name']
+
+        if name not in new_model_names:
+            continue
+
+        embedding_model_type = collection_config.get('embedding_model_type')
+        text_splitters_types = collection_config.get('text_splitters_types')
+        batch_size = collection_config.get('batch_size')
+        partial_df_size = collection_config.get('partial_dataframe_size')
+        columns = collection_config.get('columns')
+        page_content_columns = collection_config.get('page_content_columns')
+        persist_dir = f'{persist_directory}/{id}'
+
+        # Load embedding model
+        try:
+            model_kwargs = collection_config.get('model_kwargs')
+            model = get_import(embedding_model_type)(
+                model_name=name, **model_kwargs)
+        except Exception as e:
+            raise Exception(f"Unknown embedding model: {embedding_model_type}") from e
+
+        # Load text splitters
+        text_splitters = []
+        text_splitters_suffixes = []
+        for text_splitter_type in text_splitters_types:
+            try:
+                type_of_text_splitter = get_import(
+                    text_splitter_type.get("type"))
+                kwargs = text_splitter_type.get("params")
+                suffix = text_splitter_type.get("suffix")
+                if kwargs:
+                    text_splitter = type_of_text_splitter(**kwargs)
+                else:
+                    text_splitter = type_of_text_splitter()
+                text_splitters.append(text_splitter)
+                text_splitters_suffixes.append(suffix)
+            except Exception as e:
+                raise Exception(f"Unknown text splitter: {text_splitter_type}") from e
+
+        for text_splitter, suffix in zip(text_splitters, text_splitters_suffixes):
+            print(f"{id}_{suffix}")
+
+            # Prepare manager parameters
+            manager_params = {
+                "collection_name": f"{id}_{suffix}",
+                "persist_directory": persist_dir,
+                "embeddings": model,
+                "text_splitter": text_splitter,
+                "batch_size": batch_size
+            }
+
+            # Instantiate the manager class
+            vector_store_manager = vector_store_manager_type(**manager_params)
+
+            # Prepare documents
+            # You may need to use a DataFrameLoader or similar to convert df to LangChain Documents
+            loader = DataFrameLoader(
+                data_frame=df, page_content_column=page_content_columns[0]
+            )
+            docs = loader.load()
+
+            # Generate vector store
+            vector_store_manager.generate_vector_store(docs)
+
+            # Optionally persist/save
+            vector_store_manager.save(persist_dir)

{ddi_fw-0.0.247 → ddi_fw-0.0.249}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.247
+Version: 0.0.249
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>