hana-ai 1.0.25070203__tar.gz → 1.0.25070700__tar.gz

{hana_ai-1.0.25070203/src/hana_ai.egg-info → hana_ai-1.0.25070700}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hana_ai
-Version: 1.0.25070203
+Version: 1.0.25070700
 Summary: Generative AI Toolkit for SAP HANA Cloud
 Author: SAP
 License: Apache-2.0
@@ -20,6 +20,7 @@ Requires-Dist: langchain-community
 Requires-Dist: langchain-core
 Requires-Dist: langchain-experimental
 Requires-Dist: langchain-text-splitters
+Requires-Dist: langchain-hana
 Requires-Dist: pydantic
 Requires-Dist: pydantic-core
 Requires-Dist: cryptography

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hana_ai"
-version = "1.0.25070203"
+version = "1.0.25070700"
 authors = [
     {name = "SAP"}
 ]

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/requirements.txt

@@ -6,6 +6,7 @@ langchain-community
 langchain-core
 langchain-experimental
 langchain-text-splitters
+langchain-hana
 pydantic
 pydantic-core
 cryptography

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/src/hana_ai/__init__.py

@@ -2,4 +2,4 @@
 hana_ai is a Python package for AI/ML related utilities.
 """
 
-__version__ = "1.0.25070203"
+__version__ = "1.0.25070700"

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/src/hana_ai/agents/hanaml_agent_with_memory.py

@@ -10,7 +10,7 @@ The following class is available:
 #pylint: disable=ungrouped-imports, abstract-method
 import json
 import logging
-from deprecated import deprecated
+import warnings
 import pandas as pd
 #from pydantic import ValidationError
 from langchain.agents import initialize_agent, AgentType, Tool
@@ -76,7 +76,6 @@ def _get_pandas_meta(df):
         return json.dumps({"columns": columns})
     return ''
 
-@deprecated(version="1.0.25070200", reason="Use HANAMLRagAgent instead")
 class HANAMLAgentWithMemory(object):
     """
     A chatbot that can remember the chat history and use it to generate responses.
@@ -108,6 +107,9 @@ class HANAMLAgentWithMemory(object):
     >>> chatbot.run(question="Analyze the data from the table MYTEST.")
     """
     def __init__(self, llm, tools, session_id="hanaai_chat_session", n_messages=10, max_observations=5, verbose=False, **kwargs):
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn("HANAMLAgentWithMemory has been deprecated. Please use HANAMLRAGAgent instead.", DeprecationWarning, stacklevel=2)
         self.llm = llm
         self.tools = list(tools)
         self.memory = InMemoryChatMessageHistory(session_id=session_id)
@@ -194,7 +196,6 @@ class HANAMLAgentWithMemory(object):
         )
         return "Chat history has been deleted successfully."
 
-
     def run(self, question):
         """
         Chat with the chatbot.

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/src/hana_ai/agents/hanaml_rag_agent.py

@@ -13,6 +13,7 @@ from typing import Dict, List, Any, Optional, Tuple, Union
 from datetime import datetime
 import logging
 import pandas as pd
+from sqlalchemy import delete
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain.agents import Tool, AgentExecutor, create_openai_functions_agent
 from langchain.memory import ConversationBufferWindowMemory
@@ -23,6 +24,7 @@ from langchain_core.callbacks.manager import CallbackManagerForChainRun
 from langchain_core.tools import BaseTool
 from langchain_community.chat_message_histories import SQLChatMessageHistory
 from langchain_community.vectorstores import FAISS
+from langchain_community.vectorstores.hanavector import HanaDB
 
 from sentence_transformers import CrossEncoder
 from hana_ai.agents.utilities import _check_generated_cap_for_bas, _inspect_python_code
@@ -74,7 +76,7 @@ class HANAMLRAGAgent:
     def __init__(self,
                  tools: List[Tool],
                  llm: Any,
-                 vectorstore_path: str = "chat_history_vectorstore",
+                 vectorstore_path: str = "chat_history_vectorstore", # FAISS vectorstore path
                  memory_window: int = 10,
                  long_term_db: str = "sqlite:///chat_history.db",
                  long_term_memory_limit: int = 1000,
@@ -84,8 +86,13 @@ class HANAMLRAGAgent:
                  forget_percentage: float = 0.1,
                  max_iterations: int = 10,
                  cross_encoder: CrossEncoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'),
-                 rerank_k: int = 10,
+                 rerank_candidates: int = 20,
+                 rerank_k: int = 3,
                  score_threshold: float = 0.5,
+                 vector_store_type = "faiss", # 'faiss' or 'hanadb'
+                 hana_connection_context = None, # Required if vector_store is 'hanadb'
+                 hana_vector_table: str = "HANA_AI_CHAT_HISTORY",
+                 drop_existing_hana_vector_table: bool = False,
                  verbose: bool = False):
         """
         Initialize the chatbot with integrated tools and memory systems.
@@ -133,11 +140,15 @@ class HANAMLRAGAgent:
         cross_encoder : CrossEncoder
             Cross-encoder model for reranking retrieved documents.
 
-            Defaults to a pre-trained cross-encoder model 'cross-encoder/ms-marco-MiniLM-L-6-v2'.            
+            Defaults to a pre-trained cross-encoder model 'cross-encoder/ms-marco-MiniLM-L-6-v2'.
+        rerank_candidates : int
+            Number of candidate documents to retrieve for reranking.
+
+            Defaults to 20.           
         rerank_k : int
             Number of documents to retrieve for reranking.
 
-            Defaults to 10.
+            Defaults to 3.
         score_threshold : float
             Similarity score threshold for retrieval.
 
@@ -158,10 +169,15 @@ class HANAMLRAGAgent:
         self.chunk_overlap = chunk_overlap
         self.forget_percentage = forget_percentage
         self.max_iterations = max_iterations
+        self.rerank_candidates = rerank_candidates
         self.rerank_k = rerank_k
         self.score_threshold = score_threshold
         self.verbose = verbose
         self.cross_encoder = cross_encoder
+        self.vectorstore_type = vector_store_type
+        self.hana_connection_context = hana_connection_context
+        self.hana_vector_table = hana_vector_table
+        self.drop_existing_hana_vector_table = drop_existing_hana_vector_table
 
         # Initialize memory systems
         self._initialize_memory()
@@ -185,7 +201,22 @@ class HANAMLRAGAgent:
         # Initialize RAG vectorstore
         self._initialize_vectorstore()
 
-    def _initialize_vectorstore(self):
+    def delete_message_long_term_store(self, message_id) -> None:
+        """Delete a specific message by its ID"""
+        long_term_store = self.long_term_store
+        try:
+            long_term_store._create_table_if_not_exists()
+            with long_term_store._make_sync_session() as session:
+                stmt = delete(long_term_store.sql_model_class).where(
+                    long_term_store.sql_model_class.id == message_id,
+                    getattr(long_term_store.sql_model_class, long_term_store.session_id_field_name) == long_term_store.session_id
+                )
+                session.execute(stmt)
+                session.commit()
+        except Exception as e:
+            logger.error("Failed to delete message with ID %s: %s", message_id, str(e))
+
+    def _initialize_faiss_vectorstore(self):
         """Initialize or load FAISS vectorstore for long-term memory"""
         # Create embeddings service
         self.embeddings = GenAIHubEmbeddings()
@@ -207,6 +238,34 @@ class HANAMLRAGAgent:
             self.vectorstore.save_local(self.vectorstore_path)
             logger.info("Created new vectorstore at %s", self.vectorstore_path)
 
+    def _initialize_hanadb_vectorstore(self):
+        """Initialize or load HANA DB vectorstore for long-term memory"""
+        if not self.hana_connection_context:
+            raise ValueError("HANA connection context is required for HANA DB vectorstore")
+
+        # Create embeddings service
+        self.embeddings = GenAIHubEmbeddings()
+
+        # Initialize or load HANA vectorstore
+        if self.drop_existing_hana_vector_table:
+            self.hana_connection_context.drop_table(self.hana_vector_table)
+        try:
+            self.vectorstore = HanaDB(
+                embedding=self.embeddings,
+                connection=self.hana_connection_context.connection,
+                table_name=self.hana_vector_table
+            )
+            logger.info("Initialized HANA DB vectorstore with table %s", self.hana_vector_table)
+        except Exception as e:
+            logger.error("Failed to initialize HANA DB vectorstore: %s", str(e))
+            raise e
+
+    def _initialize_vectorstore(self):
+        if self.vectorstore_type.lower() == "faiss":
+            self._initialize_faiss_vectorstore()
+        else:
+            self._initialize_hanadb_vectorstore()
+
     def _should_store(self, text: str) -> bool:
         """Determine if text should be stored in memory"""
         # Skip DataFrame conversions
@@ -251,7 +310,8 @@ class HANAMLRAGAgent:
 
         # Update vector store
         self.vectorstore.add_documents(documents)
-        self.vectorstore.save_local(self.vectorstore_path)
+        if self.vectorstore_type.lower() == "faiss":
+            self.vectorstore.save_local(self.vectorstore_path)
 
         # Clean up oldest memories if needed
         self._forget_old_memories()
@@ -261,20 +321,107 @@ class HANAMLRAGAgent:
         if len(self.long_term_store.messages) > self.long_term_memory_limit:
             # Calculate number of oldest messages to remove
             num_to_remove = int(self.long_term_memory_limit * self.forget_percentage)
-
-            # Implement actual deletion logic here
-            # This is placeholder for actual database deletion
-            logger.info("Should remove %d oldest memories", num_to_remove)
-            # In practice: Delete oldest records from database
+            # 1. Delete from SQL database
+            try:
+                # Get sorted list of messages by timestamp
+                sorted_messages = sorted(
+                    self.long_term_store.messages,
+                    key=lambda msg: msg.metadata.get('timestamp', '1970-01-01')
+                )
+                # Delete oldest messages from SQL store
+                for msg in sorted_messages[:num_to_remove]:
+                    self.delete_message_long_term_store(msg.id)
+                logger.info("Deleted %s oldest memories from SQL store", num_to_remove)
+            except Exception as e:
+                logger.error("Error deleting from SQL store: %s", str(e))
+            # 2. Rebuild vectorstore from remaining messages
+            try:
+                # Get remaining messages after deletion
+                remaining_messages = self.long_term_store.messages
+                # Skip if no messages left
+                if not remaining_messages:
+                    if self.vectorstore_type.lower() == "faiss":
+                        self.vectorstore = FAISS.from_texts(
+                            [""],
+                            embedding=self.embeddings
+                        )
+                    else:
+                        self.hana_connection_context.drop_table(self.hana_vector_table)
+                        self.vectorstore = HanaDB.from_texts(
+                            texts=[""],
+                            embedding=self.embeddings,
+                            connection=self.hana_connection_context.connection,
+                            table_name=self.hana_vector_table
+                        )
+                    return
+                # Recreate documents from remaining messages
+                splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=self.chunk_size,
+                    chunk_overlap=self.chunk_overlap,
+                )
+                documents = []
+                for i in range(0, len(remaining_messages), 2):
+                    if i+1 >= len(remaining_messages):
+                        break  # Skip incomplete pairs
+                    user_msg = remaining_messages[i]
+                    ai_msg = remaining_messages[i+1]
+                    # Create combined document
+                    text = f"User: {user_msg.content}\nAssistant: {ai_msg.content}"
+                    metadata = {"timestamp": user_msg.metadata.get('timestamp', '')}
+                    # Split and add to documents
+                    docs = splitter.create_documents(
+                        texts=[text],
+                        metadatas=[metadata]
+                    )
+                    documents.extend(docs)
+                # Rebuild vectorstore
+                if self.vectorstore_type.lower() == "faiss":
+                    self.vectorstore = FAISS.from_documents(
+                        documents,
+                        self.embeddings
+                    )
+                    self.vectorstore.save_local(self.vectorstore_path)
+                else:
+                    self.hana_connection_context.drop_table(self.hana_vector_table)
+                    self.vectorstore = HanaDB.from_documents(
+                        documents,
+                        self.embeddings,
+                        connection=self.hana_connection_context.connection,
+                        table_name=self.hana_vector_table
+                    )
+                logger.info("Rebuilt vectorstore with %s documents", len(documents))
+            except Exception as e:
+                logger.error("Error rebuilding vectorstore: %s", str(e))
+                # Fallback to empty vectorstore
+                if self.vectorstore_type.lower() == "faiss":
+                    self.vectorstore = FAISS.from_texts(
+                        [""],
+                        embedding=self.embeddings
+                    )
+                else:
+                    self.hana_connection_context.drop_table(self.hana_vector_table)
+                    self.vectorstore = HanaDB.from_texts(
+                        texts=[""],
+                        embedding=self.embeddings,
+                        connection=self.hana_connection_context.connection,
+                        table_name=self.hana_vector_table
+                    )
 
     def _retrieve_relevant_memories(self, query: str) -> List[str]:
         """Retrieve relevant memories using RAG with reranking"""
         # Retrieve candidate documents
-        candidate_docs = self.vectorstore.similarity_search_with_score(
-            query=query,
-            k=self.rerank_k,
-            score_threshold=self.score_threshold
-        )
+        if self.vectorstore_type.lower() == "faiss":
+            candidate_docs = self.vectorstore.similarity_search_with_score(
+                query=query,
+                k=self.rerank_candidates,
+                score_threshold=self.score_threshold
+            )
+        else:
+            candidate_docs = self.vectorstore.similarity_search_with_relevance_scores(
+                query=query,
+                k=self.rerank_candidates,
+                score_threshold=self.score_threshold
+            )
 
         if not candidate_docs:
             return []
@@ -293,7 +440,7 @@ class HANAMLRAGAgent:
         combined = list(zip(doc_contents, rerank_scores))
         combined.sort(key=lambda x: x[1], reverse=True)
 
-        return [content for content, _ in combined[:3]]
+        return [content for content, _ in combined[:self.rerank_k]]
 
     def _build_context(self, user_input: str) -> SystemMessage:
         # 获取短时记忆并转换格式
@@ -368,8 +515,17 @@ class HANAMLRAGAgent:
             logger.error("Unexpected error clearing SQL store: %s", str(e))
 
         # 重建空向量库（无需前置检查）
-        self.vectorstore = FAISS.from_texts([""], embedding=self.embeddings)  # 空文本占位
-        self.vectorstore.save_local(self.vectorstore_path)
+        if self.vectorstore_type.lower() == "faiss":
+            self.vectorstore = FAISS.from_texts([""], embedding=self.embeddings)  # 空文本占位
+            self.vectorstore.save_local(self.vectorstore_path)
+        else:
+            self.hana_connection_context.drop_table(self.hana_vector_table)
+            self.vectorstore = HanaDB.from_texts(
+                texts=[""],
+                embedding=self.embeddings,
+                connection=self.hana_connection_context.connection,
+                table_name=self.hana_vector_table
+            )
         logger.info("Long-term memory cleared and vectorstore reset.")
 
     def clear_short_term_memory(self):

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/src/hana_ai/tools/hana_ml_tools/fetch_tools.py

@@ -96,7 +96,7 @@ class FetchDataTool(BaseTool):
         else:
             results = self.connection_context.table(table_name).collect()
         if not self.return_direct:
-            results = results.to_string(index=False)
+            results = results.to_markdown(index=False)
         return results
 
     async def _arun(

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700/src/hana_ai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hana_ai
-Version: 1.0.25070203
+Version: 1.0.25070700
 Summary: Generative AI Toolkit for SAP HANA Cloud
 Author: SAP
 License: Apache-2.0
@@ -20,6 +20,7 @@ Requires-Dist: langchain-community
 Requires-Dist: langchain-core
 Requires-Dist: langchain-experimental
 Requires-Dist: langchain-text-splitters
+Requires-Dist: langchain-hana
 Requires-Dist: pydantic
 Requires-Dist: pydantic-core
 Requires-Dist: cryptography

{hana_ai-1.0.25070203 → hana_ai-1.0.25070700}/src/hana_ai.egg-info/requires.txt

@@ -6,6 +6,7 @@ langchain-community
 langchain-core
 langchain-experimental
 langchain-text-splitters
+langchain-hana
 pydantic
 pydantic-core
 cryptography