vanna 0.6.5__py3-none-any.whl → 0.6.6__py3-none-any.whl

vanna/base/base.py

@@ -437,7 +437,7 @@ class VannaBase(ABC):
         pass
 
     @abstractmethod
-    def remove_training_data(id: str, **kwargs) -> bool:
+    def remove_training_data(self, id: str, **kwargs) -> bool:
         """
         Example:
         ```python
@@ -1276,15 +1276,10 @@ class VannaBase(ABC):
 
         def run_sql_bigquery(sql: str) -> Union[pd.DataFrame, None]:
             if conn:
-                try:
-                    job = conn.query(sql)
-                    df = job.result().to_dataframe()
-                    return df
-                except GoogleAPIError as error:
-                    errors = []
-                    for error in error.errors:
-                        errors.append(error["message"])
-                    raise errors
+                job = conn.query(sql)
+                df = job.result().to_dataframe()
+                return df
+
             return None
 
         self.dialect = "BigQuery SQL"

vanna/flask/__init__.py

@@ -12,9 +12,9 @@ from flasgger import Swagger
 from flask import Flask, Response, jsonify, request, send_from_directory
 from flask_sock import Sock
 
+from ..base import VannaBase
 from .assets import css_content, html_content, js_content
 from .auth import AuthInterface, NoAuth
-from ..base import VannaBase
 
 
 class Cache(ABC):
@@ -1211,7 +1211,7 @@ class VannaFlaskApp(VannaFlaskAPI):
         self.config["ask_results_correct"] = ask_results_correct
         self.config["followup_questions"] = followup_questions
         self.config["summarization"] = summarization
-        self.config["function_generation"] = function_generation
+        self.config["function_generation"] = function_generation and hasattr(vn, "get_function")
 
         self.index_html_path = index_html_path
         self.assets_folder = assets_folder

vanna/google/__init__.py

@@ -1 +1,2 @@
-from .gemini_chat import GoogleGeminiChat
+from .bigquery_vector import BigQuery_VectorStore
+from .gemini_chat import GoogleGeminiChat

vanna/google/bigquery_vector.py

@@ -0,0 +1,230 @@
+import datetime
+import os
+import uuid
+from typing import List, Optional
+
+import pandas as pd
+from google.cloud import bigquery
+
+from ..base import VannaBase
+
+
+class BigQuery_VectorStore(VannaBase):
+    def __init__(self, config: dict, **kwargs):
+        self.config = config
+
+        self.n_results_sql = config.get("n_results_sql", config.get("n_results", 10))
+        self.n_results_documentation = config.get("n_results_documentation", config.get("n_results", 10))
+        self.n_results_ddl = config.get("n_results_ddl", config.get("n_results", 10))
+
+        if "api_key" in config or os.getenv("GOOGLE_API_KEY"):
+            """
+            If Google api_key is provided through config
+            or set as an environment variable, assign it.
+            """
+            print("Configuring genai")
+            import google.generativeai as genai
+
+            genai.configure(api_key=config["api_key"])
+
+            self.genai = genai
+        else:
+            # Authenticate using VertexAI
+            from vertexai.language_models import (
+              TextEmbeddingInput,
+              TextEmbeddingModel,
+            )
+
+        if self.config.get("project_id"):
+            self.project_id = self.config.get("project_id")
+        else:
+            self.project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+        if self.project_id is None:
+            raise ValueError("Project ID is not set")
+
+        self.conn = bigquery.Client(project=self.project_id)
+
+        dataset_name = self.config.get('bigquery_dataset_name', 'vanna_managed')
+        self.dataset_id = f"{self.project_id}.{dataset_name}"
+        dataset = bigquery.Dataset(self.dataset_id)
+
+        try:
+            self.conn.get_dataset(self.dataset_id)  # Make an API request.
+            print(f"Dataset {self.dataset_id} already exists")
+        except Exception:
+            # Dataset does not exist, create it
+            dataset.location = "US"
+            self.conn.create_dataset(dataset, timeout=30)  # Make an API request.
+            print(f"Created dataset {self.dataset_id}")
+
+        # Create a table called training_data in the dataset that contains the columns:
+        # id, training_data_type, question, content, embedding, created_at
+
+        self.table_id = f"{self.dataset_id}.training_data"
+        schema = [
+            bigquery.SchemaField("id", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("training_data_type", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("question", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("content", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("embedding", "FLOAT64", mode="REPEATED"),
+            bigquery.SchemaField("created_at", "TIMESTAMP", mode="REQUIRED"),
+        ]
+
+        table = bigquery.Table(self.table_id, schema=schema)
+
+        try:
+            self.conn.get_table(self.table_id)  # Make an API request.
+            print(f"Table {self.table_id} already exists")
+        except Exception:
+            # Table does not exist, create it
+            self.conn.create_table(table, timeout=30)  # Make an API request.
+            print(f"Created table {self.table_id}")
+
+        # Create VECTOR INDEX IF NOT EXISTS
+        # TODO: This requires 5000 rows before it can be created
+        # vector_index_query = f"""
+        # CREATE VECTOR INDEX IF NOT EXISTS my_index
+        # ON `{self.table_id}`(embedding)
+        # OPTIONS(
+        #     distance_type='COSINE',
+        #     index_type='IVF',
+        #     ivf_options='{{"num_lists": 1000}}'
+        # )
+        # """
+
+        # try:
+        #     self.conn.query(vector_index_query).result()  # Make an API request.
+        #     print(f"Vector index on {self.table_id} created or already exists")
+        # except Exception as e:
+        #     print(f"Failed to create vector index: {e}")
+
+    def store_training_data(self, training_data_type: str, question: str, content: str, embedding: List[float], **kwargs) -> str:
+        id = str(uuid.uuid4())
+        created_at = datetime.datetime.now()
+        self.conn.insert_rows_json(self.table_id, [{
+            "id": id,
+            "training_data_type": training_data_type,
+            "question": question,
+            "content": content,
+            "embedding": embedding,
+            "created_at": created_at.isoformat()
+        }])
+
+        return id
+
+    def fetch_similar_training_data(self, training_data_type: str, question: str, n_results, **kwargs) -> pd.DataFrame:
+        question_embedding = self.generate_question_embedding(question)
+
+        query = f"""
+        SELECT
+            base.id as id,
+            base.question as question,
+            base.training_data_type as training_data_type,
+            base.content as content,
+            distance
+        FROM
+            VECTOR_SEARCH(
+                TABLE `{self.table_id}`,
+                'embedding',
+                (SELECT * FROM UNNEST([STRUCT({question_embedding})])),
+                top_k => 5,
+                distance_type => 'COSINE',
+                options => '{{"use_brute_force":true}}'
+            )
+        WHERE
+            base.training_data_type = '{training_data_type}'
+        """
+
+        results = self.conn.query(query).result().to_dataframe()
+        return results
+
+    def generate_question_embedding(self, data: str, **kwargs) -> List[float]:
+        result = self.genai.embed_content(
+            model="models/text-embedding-004",
+            content=data,
+            task_type="retrieval_query")
+
+        if 'embedding' in result:
+            return result['embedding']
+        else:
+            raise ValueError("No embeddings returned")
+
+    def generate_storage_embedding(self, data: str, **kwargs) -> List[float]:
+        result = self.genai.embed_content(
+            model="models/text-embedding-004",
+            content=data,
+            task_type="retrieval_document")
+
+        if 'embedding' in result:
+            return result['embedding']
+        else:
+            raise ValueError("No embeddings returned")
+
+        # task = "RETRIEVAL_DOCUMENT"
+        # inputs = [TextEmbeddingInput(data, task)]
+        # embeddings = self.vertex_embedding_model.get_embeddings(inputs)
+
+        # if len(embeddings) == 0:
+        #     raise ValueError("No embeddings returned")
+
+        # return embeddings[0].values
+
+        return result
+
+    def generate_embedding(self, data: str, **kwargs) -> List[float]:
+        return self.generate_storage_embedding(data, **kwargs)
+
+    def get_similar_question_sql(self, question: str, **kwargs) -> list:
+        df = self.fetch_similar_training_data(training_data_type="sql", question=question, n_results=self.n_results_sql)
+
+        # Return a list of dictionaries with only question, sql fields. The content field needs to be renamed to sql
+        return df.rename(columns={"content": "sql"})[["question", "sql"]].to_dict(orient="records")
+
+    def get_related_ddl(self, question: str, **kwargs) -> list:
+        df = self.fetch_similar_training_data(training_data_type="ddl", question=question, n_results=self.n_results_ddl)
+
+        # Return a list of strings of the content
+        return df["content"].tolist()
+
+    def get_related_documentation(self, question: str, **kwargs) -> list:
+        df = self.fetch_similar_training_data(training_data_type="documentation", question=question, n_results=self.n_results_documentation)
+
+        # Return a list of strings of the content
+        return df["content"].tolist()
+
+    def add_question_sql(self, question: str, sql: str, **kwargs) -> str:
+        doc = {
+            "question": question,
+            "sql": sql
+        }
+
+        embedding = self.generate_embedding(str(doc))
+
+        return self.store_training_data(training_data_type="sql", question=question, content=sql, embedding=embedding)
+
+    def add_ddl(self, ddl: str, **kwargs) -> str:
+        embedding = self.generate_embedding(ddl)
+
+        return self.store_training_data(training_data_type="ddl", question="", content=ddl, embedding=embedding)
+
+    def add_documentation(self, documentation: str, **kwargs) -> str:
+        embedding = self.generate_embedding(documentation)
+
+        return self.store_training_data(training_data_type="documentation", question="", content=documentation, embedding=embedding)
+
+    def get_training_data(self, **kwargs) -> pd.DataFrame:
+        query = f"SELECT id, training_data_type, question, content FROM `{self.table_id}`"
+
+        return self.conn.query(query).result().to_dataframe()
+
+    def remove_training_data(self, id: str, **kwargs) -> bool:
+        query = f"DELETE FROM `{self.table_id}` WHERE id = '{id}'"
+
+        try:
+            self.conn.query(query).result()
+            return True
+
+        except Exception as e:
+            print(f"Failed to remove training data: {e}")
+            return False

vanna/mistral/mistral.py

@@ -1,5 +1,7 @@
-from mistralai.client import MistralClient
-from mistralai.models.chat_completion import ChatMessage
+import os
+
+from mistralai import Mistral as MistralClient
+from mistralai import UserMessage
 
 from ..base import VannaBase
 
@@ -23,13 +25,13 @@ class Mistral(VannaBase):
         self.model = model
 
     def system_message(self, message: str) -> any:
-        return ChatMessage(role="system", content=message)
+        return {"role": "system", "content": message}
 
     def user_message(self, message: str) -> any:
-        return ChatMessage(role="user", content=message)
+        return {"role": "user", "content": message}
 
     def assistant_message(self, message: str) -> any:
-        return ChatMessage(role="assistant", content=message)
+        return {"role": "assistant", "content": message}
 
     def generate_sql(self, question: str, **kwargs) -> str:
         # Use the super generate_sql
@@ -41,7 +43,7 @@ class Mistral(VannaBase):
         return sql
 
     def submit_prompt(self, prompt, **kwargs) -> str:
-        chat_response = self.client.chat(
+        chat_response = self.client.chat.complete(
             model=self.model,
             messages=prompt,
         )

{vanna-0.6.5.dist-info → vanna-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vanna
-Version: 0.6.5
+Version: 0.6.6
 Summary: Generate SQL queries from natural language
 Author-email: Zain Hoda <zain@vanna.ai>
 Requires-Python: >=3.9
@@ -26,7 +26,7 @@ Requires-Dist: snowflake-connector-python ; extra == "all"
 Requires-Dist: duckdb ; extra == "all"
 Requires-Dist: openai ; extra == "all"
 Requires-Dist: qianfan ; extra == "all"
-Requires-Dist: mistralai ; extra == "all"
+Requires-Dist: mistralai>=1.0.0 ; extra == "all"
 Requires-Dist: chromadb ; extra == "all"
 Requires-Dist: anthropic ; extra == "all"
 Requires-Dist: zhipuai ; extra == "all"
@@ -56,7 +56,7 @@ Requires-Dist: google-cloud-aiplatform ; extra == "google"
 Requires-Dist: transformers ; extra == "hf"
 Requires-Dist: marqo ; extra == "marqo"
 Requires-Dist: pymilvus[model] ; extra == "milvus"
-Requires-Dist: mistralai ; extra == "mistralai"
+Requires-Dist: mistralai>=1.0.0 ; extra == "mistralai"
 Requires-Dist: PyMySQL ; extra == "mysql"
 Requires-Dist: ollama ; extra == "ollama"
 Requires-Dist: httpx ; extra == "ollama"

{vanna-0.6.5.dist-info → vanna-0.6.6.dist-info}/RECORD

@@ -9,16 +9,17 @@ vanna/advanced/__init__.py,sha256=oDj9g1JbrbCfp4WWdlr_bhgdMqNleyHgr6VXX6DcEbo,65
 vanna/anthropic/__init__.py,sha256=85s_2mAyyPxc0T_0JEvYeAkEKWJwkwqoyUwSC5dw9Gk,43
 vanna/anthropic/anthropic_chat.py,sha256=7X3x8SYwDY28aGyBnt0YNRMG8YY1p_t-foMfKGj8_Oo,2627
 vanna/base/__init__.py,sha256=Sl-HM1RRYzAZoSqmL1CZQmF3ZF-byYTCFQP3JZ2A5MU,28
-vanna/base/base.py,sha256=3Du70NrXQMn_LOif2YFPRRVKo4wH5-f6eZcLlXEX0X8,71705
+vanna/base/base.py,sha256=DrXaJcMhIjD6BEqLu4JNZaZZ8nTM4SppjdvueJjEcko,71463
 vanna/bedrock/__init__.py,sha256=hRT2bgJbHEqViLdL-t9hfjSfFdIOkPU2ADBt-B1En-8,46
 vanna/bedrock/bedrock_converse.py,sha256=Nx5kYm-diAfYmsWAnTP5xnv7V84Og69-AP9b3seIe0E,2869
 vanna/chromadb/__init__.py,sha256=-iL0nW_g4uM8nWKMuWnNePfN4nb9uk8P3WzGvezOqRg,50
 vanna/chromadb/chromadb_vector.py,sha256=eKyPck99Y6Jt-BNWojvxLG-zvAERzLSm-3zY-bKXvaA,8792
 vanna/exceptions/__init__.py,sha256=dJ65xxxZh1lqBeg6nz6Tq_r34jLVmjvBvPO9Q6hFaQ8,685
-vanna/flask/__init__.py,sha256=r1ucQupb6wuTcjVVKpkdrg6R38FZe6KQoKw9AtcghDQ,42889
+vanna/flask/__init__.py,sha256=cllLWqJ2SYVdvF4CQ-8cipoOdjgkoe0rChWnawtzMyA,42921
 vanna/flask/assets.py,sha256=_UoUr57sS0QL2BuTxAOe9k4yy8T7-fp2NpbRSVtW3IM,451769
 vanna/flask/auth.py,sha256=UpKxh7W5cd43W0LGch0VqhncKwB78L6dtOQkl1JY5T0,1246
-vanna/google/__init__.py,sha256=M-dCxCZcKL4bTQyMLj6r6VRs65YNX9Tl2aoPCuqGm-8,41
+vanna/google/__init__.py,sha256=6D8rDBjKJJm_jpVn9b4Vc2NR-R779ed_bnHhWmxCJXE,92
+vanna/google/bigquery_vector.py,sha256=rkP94Xd1lNYjU1x3MDLvqmGSPUYtDfQwvlqVmX44jyM,8839
 vanna/google/gemini_chat.py,sha256=j1szC2PamMLFrs0Z4lYPS69i017FYICe-mNObNYFBPQ,1576
 vanna/hf/__init__.py,sha256=vD0bIhfLkA1UsvVSF4MAz3Da8aQunkQo3wlDztmMuj0,19
 vanna/hf/hf.py,sha256=N8N5g3xvKDBt3dez2r_U0qATxbl2pN8SVLTZK9CSRA0,3020
@@ -27,7 +28,7 @@ vanna/marqo/marqo.py,sha256=W7WTtzWp4RJjZVy6OaXHqncUBIPdI4Q7qH7BRCxZ1_A,5242
 vanna/milvus/__init__.py,sha256=VBasJG2eTKbJI6CEand7kPLNBrqYrn0QCAhSYVz814s,46
 vanna/milvus/milvus_vector.py,sha256=Mq0eaSh0UcTYhgh8mTm0fvS6rbfL6tQONVnDZGemWoM,11268
 vanna/mistral/__init__.py,sha256=70rTY-69Z2ehkkMj84dNMCukPo6AWdflBGvIB_pztS0,29
-vanna/mistral/mistral.py,sha256=DAEqAT9SzC91rfMM_S3SuzBZ34MrKHw9qAj6EP2MGVk,1508
+vanna/mistral/mistral.py,sha256=rcdgmUSQniLkah2VL23VGYRa9WXpOy_dZN4S0kc__V8,1494
 vanna/mock/__init__.py,sha256=nYR2WfcV5NdwpK3V64QGOWHBGc3ESN9uV68JLS76aRw,97
 vanna/mock/embedding.py,sha256=ggnP7KuPh6dlqeUFtoN8t0J0P7_yRNtn9rIq6h8g8-w,250
 vanna/mock/llm.py,sha256=WpG9f1pKZftPBHqgIYdARKB2Z9DZhOALYOJWoOjjFEc,518
@@ -53,6 +54,6 @@ vanna/vllm/__init__.py,sha256=aNlUkF9tbURdeXAJ8ytuaaF1gYwcG3ny1MfNl_cwQYg,23
 vanna/vllm/vllm.py,sha256=oM_aA-1Chyl7T_Qc_yRKlL6oSX1etsijY9zQdjeMGMQ,2827
 vanna/weaviate/__init__.py,sha256=HL6PAl7ePBAkeG8uln-BmM7IUtWohyTPvDfcPzSGSCg,46
 vanna/weaviate/weaviate_vector.py,sha256=GEiu4Vd9w-7j10aB-zTxJ8gefqe_F-LUUGvttFs1vlg,7539
-vanna-0.6.5.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-vanna-0.6.5.dist-info/METADATA,sha256=77ggtzQplTTDdsOI0_U4k-t5UwHVmI5O3TSkpm5OXzY,11997
-vanna-0.6.5.dist-info/RECORD,,
+vanna-0.6.6.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+vanna-0.6.6.dist-info/METADATA,sha256=_qy1wVZqQOLplCxZ43KqCIEgL0Wq2X48ekJEACq-0Ng,12011
+vanna-0.6.6.dist-info/RECORD,,