vanna 0.6.0__tar.gz → 0.6.2__tar.gz

{vanna-0.6.0 → vanna-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vanna
-Version: 0.6.0
+Version: 0.6.2
 Summary: Generate SQL queries from natural language
 Author-email: Zain Hoda <zain@vanna.ai>
 Requires-Python: >=3.9
@@ -39,6 +39,7 @@ Requires-Dist: opensearch-py ; extra == "all"
 Requires-Dist: opensearch-dsl ; extra == "all"
 Requires-Dist: transformers ; extra == "all"
 Requires-Dist: pinecone-client ; extra == "all"
+Requires-Dist: pymilvus[model] ; extra == "all"
 Requires-Dist: anthropic ; extra == "anthropic"
 Requires-Dist: google-cloud-bigquery ; extra == "bigquery"
 Requires-Dist: chromadb ; extra == "chromadb"
@@ -49,6 +50,7 @@ Requires-Dist: google-generativeai ; extra == "google"
 Requires-Dist: google-cloud-aiplatform ; extra == "google"
 Requires-Dist: transformers ; extra == "hf"
 Requires-Dist: marqo ; extra == "marqo"
+Requires-Dist: pymilvus[model] ; extra == "milvus"
 Requires-Dist: mistralai ; extra == "mistralai"
 Requires-Dist: PyMySQL ; extra == "mysql"
 Requires-Dist: ollama ; extra == "ollama"
@@ -78,6 +80,7 @@ Provides-Extra: gemini
 Provides-Extra: google
 Provides-Extra: hf
 Provides-Extra: marqo
+Provides-Extra: milvus
 Provides-Extra: mistralai
 Provides-Extra: mysql
 Provides-Extra: ollama

{vanna-0.6.0 → vanna-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "vanna"
-version = "0.6.0"
+version = "0.6.2"
 authors = [
   { name="Zain Hoda", email="zain@vanna.ai" },
 ]
@@ -33,7 +33,7 @@ bigquery = ["google-cloud-bigquery"]
 snowflake = ["snowflake-connector-python"]
 duckdb = ["duckdb"]
 google = ["google-generativeai", "google-cloud-aiplatform"]
-all = ["psycopg2-binary", "db-dtypes", "PyMySQL", "google-cloud-bigquery", "snowflake-connector-python", "duckdb", "openai", "mistralai", "chromadb", "anthropic", "zhipuai", "marqo", "google-generativeai", "google-cloud-aiplatform", "qdrant-client", "fastembed", "ollama", "httpx", "opensearch-py", "opensearch-dsl", "transformers", "pinecone-client"]
+all = ["psycopg2-binary", "db-dtypes", "PyMySQL", "google-cloud-bigquery", "snowflake-connector-python", "duckdb", "openai", "mistralai", "chromadb", "anthropic", "zhipuai", "marqo", "google-generativeai", "google-cloud-aiplatform", "qdrant-client", "fastembed", "ollama", "httpx", "opensearch-py", "opensearch-dsl", "transformers", "pinecone-client", "pymilvus[model]"]
 test = ["tox"]
 chromadb = ["chromadb"]
 openai = ["openai"]
@@ -48,3 +48,4 @@ vllm = ["vllm"]
 pinecone = ["pinecone-client", "fastembed"]
 opensearch = ["opensearch-py", "opensearch-dsl"]
 hf = ["transformers"]
+milvus = ["pymilvus[model]"]

{vanna-0.6.0 → vanna-0.6.2}/src/vanna/base/base.py

@@ -992,6 +992,7 @@ class VannaBase(ABC):
         def run_sql_mysql(sql: str) -> Union[pd.DataFrame, None]:
             if conn:
                 try:
+                    conn.ping(reconnect=True)
                     cs = conn.cursor()
                     cs.execute(sql)
                     results = cs.fetchall()

{vanna-0.6.0 → vanna-0.6.2}/src/vanna/flask/__init__.py

@@ -494,15 +494,12 @@ class VannaFlaskApp:
         def generate_plotly_figure(user: any, id: str, df, question, sql):
             chart_instructions = flask.request.args.get('chart_instructions')
 
-            if chart_instructions is not None:
-                question = f"{question}. When generating the chart, use these special instructions: {chart_instructions}"
-
             try:
                 # If chart_instructions is not set then attempt to retrieve the code from the cache
                 if chart_instructions is None or len(chart_instructions) == 0:
                     code = self.cache.get(id=id, field="plotly_code")
-
-                if code is None:
+                else:
+                    question = f"{question}. When generating the chart, use these special instructions: {chart_instructions}"
                     code = vn.generate_plotly_code(
                         question=question,
                         sql=sql,

vanna-0.6.2/src/vanna/milvus/__init__.py

@@ -0,0 +1 @@
+from .milvus_vector import Milvus_VectorStore

vanna-0.6.2/src/vanna/milvus/milvus_vector.py

@@ -0,0 +1,305 @@
+import uuid
+from typing import List
+
+import pandas as pd
+from pymilvus import DataType, MilvusClient, model
+
+from ..base import VannaBase
+
+# Setting the URI as a local file, e.g.`./milvus.db`,
+# is the most convenient method, as it automatically utilizes Milvus Lite
+# to store all data in this file.
+#
+# If you have large scale of data such as more than a million docs, we
+# recommend setting up a more performant Milvus server on docker or kubernetes.
+# When using this setup, please use the server URI,
+# e.g.`http://localhost:19530`, as your URI.
+
+DEFAULT_MILVUS_URI = "./milvus.db"
+# DEFAULT_MILVUS_URI = "http://localhost:19530"
+
+MAX_LIMIT_SIZE = 10_000
+
+
+class Milvus_VectorStore(VannaBase):
+    """
+    Vectorstore implementation using Milvus - https://milvus.io/docs/quickstart.md
+
+    Args:
+        - config (dict, optional): Dictionary of `Milvus_VectorStore config` options. Defaults to `None`.
+            - milvus_client: A `pymilvus.MilvusClient` instance.
+            - embedding_function:
+                A `milvus_model.base.BaseEmbeddingFunction` instance. Defaults to `DefaultEmbeddingFunction()`.
+                For more models, please refer to:
+                https://milvus.io/docs/embeddings.md
+    """
+    def __init__(self, config=None):
+        VannaBase.__init__(self, config=config)
+
+        if "milvus_client" in config:
+            self.milvus_client = config["milvus_client"]
+        else:
+            self.milvus_client = MilvusClient(uri=DEFAULT_MILVUS_URI)
+
+        if "embedding_function" in config:
+            self.embedding_function = config.get("embedding_function")
+        else:
+            self.embedding_function = model.DefaultEmbeddingFunction()
+        self._embedding_dim = self.embedding_function.encode_documents(["foo"])[0].shape[0]
+        self._create_collections()
+        self.n_results = config.get("n_results", 10)
+
+    def _create_collections(self):
+        self._create_sql_collection("vannasql")
+        self._create_ddl_collection("vannaddl")
+        self._create_doc_collection("vannadoc")
+
+
+    def generate_embedding(self, data: str, **kwargs) -> List[float]:
+        return self.embedding_function.encode_documents(data).tolist()
+
+
+    def _create_sql_collection(self, name: str):
+        if not self.milvus_client.has_collection(collection_name=name):
+            vannasql_schema = MilvusClient.create_schema(
+                auto_id=False,
+                enable_dynamic_field=False,
+            )
+            vannasql_schema.add_field(field_name="id", datatype=DataType.VARCHAR, max_length=65535, is_primary=True)
+            vannasql_schema.add_field(field_name="text", datatype=DataType.VARCHAR, max_length=65535)
+            vannasql_schema.add_field(field_name="sql", datatype=DataType.VARCHAR, max_length=65535)
+            vannasql_schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self._embedding_dim)
+
+            vannasql_index_params = self.milvus_client.prepare_index_params()
+            vannasql_index_params.add_index(
+                field_name="vector",
+                index_name="vector",
+                index_type="AUTOINDEX",
+                metric_type="L2",
+            )
+            self.milvus_client.create_collection(
+                collection_name=name,
+                schema=vannasql_schema,
+                index_params=vannasql_index_params,
+                consistency_level="Strong"
+            )
+
+    def _create_ddl_collection(self, name: str):
+        if not self.milvus_client.has_collection(collection_name=name):
+            vannaddl_schema = MilvusClient.create_schema(
+                auto_id=False,
+                enable_dynamic_field=False,
+            )
+            vannaddl_schema.add_field(field_name="id", datatype=DataType.VARCHAR, max_length=65535, is_primary=True)
+            vannaddl_schema.add_field(field_name="ddl", datatype=DataType.VARCHAR, max_length=65535)
+            vannaddl_schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self._embedding_dim)
+
+            vannaddl_index_params = self.milvus_client.prepare_index_params()
+            vannaddl_index_params.add_index(
+                field_name="vector",
+                index_name="vector",
+                index_type="AUTOINDEX",
+                metric_type="L2",
+            )
+            self.milvus_client.create_collection(
+                collection_name=name,
+                schema=vannaddl_schema,
+                index_params=vannaddl_index_params,
+                consistency_level="Strong"
+            )
+
+    def _create_doc_collection(self, name: str):
+        if not self.milvus_client.has_collection(collection_name=name):
+            vannadoc_schema = MilvusClient.create_schema(
+                auto_id=False,
+                enable_dynamic_field=False,
+            )
+            vannadoc_schema.add_field(field_name="id", datatype=DataType.VARCHAR, max_length=65535, is_primary=True)
+            vannadoc_schema.add_field(field_name="doc", datatype=DataType.VARCHAR, max_length=65535)
+            vannadoc_schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self._embedding_dim)
+
+            vannadoc_index_params = self.milvus_client.prepare_index_params()
+            vannadoc_index_params.add_index(
+                field_name="vector",
+                index_name="vector",
+                index_type="AUTOINDEX",
+                metric_type="L2",
+            )
+            self.milvus_client.create_collection(
+                collection_name=name,
+                schema=vannadoc_schema,
+                index_params=vannadoc_index_params,
+                consistency_level="Strong"
+            )
+
+    def add_question_sql(self, question: str, sql: str, **kwargs) -> str:
+        if len(question) == 0 or len(sql) == 0:
+            raise Exception("pair of question and sql can not be null")
+        _id = str(uuid.uuid4()) + "-sql"
+        embedding = self.embedding_function.encode_documents([question])[0]
+        self.milvus_client.insert(
+            collection_name="vannasql",
+            data={
+                "id": _id,
+                "text": question,
+                "sql": sql,
+                "vector": embedding
+            }
+        )
+        return _id
+
+    def add_ddl(self, ddl: str, **kwargs) -> str:
+        if len(ddl) == 0:
+            raise Exception("ddl can not be null")
+        _id = str(uuid.uuid4()) + "-ddl"
+        embedding = self.embedding_function.encode_documents([ddl])[0]
+        self.milvus_client.insert(
+            collection_name="vannaddl",
+            data={
+                "id": _id,
+                "ddl": ddl,
+                "vector": embedding
+            }
+        )
+        return _id
+
+    def add_documentation(self, documentation: str, **kwargs) -> str:
+        if len(documentation) == 0:
+            raise Exception("documentation can not be null")
+        _id = str(uuid.uuid4()) + "-doc"
+        embedding = self.embedding_function.encode_documents([documentation])[0]
+        self.milvus_client.insert(
+            collection_name="vannadoc",
+            data={
+                "id": _id,
+                "doc": documentation,
+                "vector": embedding
+            }
+        )
+        return _id
+
+    def get_training_data(self, **kwargs) -> pd.DataFrame:
+        sql_data = self.milvus_client.query(
+            collection_name="vannasql",
+            output_fields=["*"],
+            limit=MAX_LIMIT_SIZE,
+        )
+        df = pd.DataFrame()
+        df_sql = pd.DataFrame(
+            {
+                "id": [doc["id"] for doc in sql_data],
+                "question": [doc["text"] for doc in sql_data],
+                "content": [doc["sql"] for doc in sql_data],
+            }
+        )
+        df = pd.concat([df, df_sql])
+
+        ddl_data = self.milvus_client.query(
+            collection_name="vannaddl",
+            output_fields=["*"],
+            limit=MAX_LIMIT_SIZE,
+        )
+
+        df_ddl = pd.DataFrame(
+            {
+                "id": [doc["id"] for doc in ddl_data],
+                "question": [None for doc in ddl_data],
+                "content": [doc["ddl"] for doc in ddl_data],
+            }
+        )
+        df = pd.concat([df, df_ddl])
+
+        doc_data = self.milvus_client.query(
+            collection_name="vannadoc",
+            output_fields=["*"],
+            limit=MAX_LIMIT_SIZE,
+        )
+
+        df_doc = pd.DataFrame(
+            {
+                "id": [doc["id"] for doc in doc_data],
+                "question": [None for doc in doc_data],
+                "content": [doc["doc"] for doc in doc_data],
+            }
+        )
+        df = pd.concat([df, df_doc])
+        return df
+
+    def get_similar_question_sql(self, question: str, **kwargs) -> list:
+        search_params = {
+            "metric_type": "L2",
+            "params": {"nprobe": 128},
+        }
+        embeddings = self.embedding_function.encode_queries([question])
+        res = self.milvus_client.search(
+            collection_name="vannasql",
+            anns_field="vector",
+            data=embeddings,
+            limit=self.n_results,
+            output_fields=["text", "sql"],
+            search_params=search_params
+        )
+        res = res[0]
+
+        list_sql = []
+        for doc in res:
+            dict = {}
+            dict["question"] = doc["entity"]["text"]
+            dict["sql"] = doc["entity"]["sql"]
+            list_sql.append(dict)
+        return list_sql
+
+    def get_related_ddl(self, question: str, **kwargs) -> list:
+        search_params = {
+            "metric_type": "L2",
+            "params": {"nprobe": 128},
+        }
+        embeddings = self.embedding_function.encode_queries([question])
+        res = self.milvus_client.search(
+            collection_name="vannaddl",
+            anns_field="vector",
+            data=embeddings,
+            limit=self.n_results,
+            output_fields=["ddl"],
+            search_params=search_params
+        )
+        res = res[0]
+
+        list_ddl = []
+        for doc in res:
+            list_ddl.append(doc["entity"]["ddl"])
+        return list_ddl
+
+    def get_related_documentation(self, question: str, **kwargs) -> list:
+        search_params = {
+            "metric_type": "L2",
+            "params": {"nprobe": 128},
+        }
+        embeddings = self.embedding_function.encode_queries([question])
+        res = self.milvus_client.search(
+            collection_name="vannadoc",
+            anns_field="vector",
+            data=embeddings,
+            limit=self.n_results,
+            output_fields=["doc"],
+            search_params=search_params
+        )
+        res = res[0]
+
+        list_doc = []
+        for doc in res:
+            list_doc.append(doc["entity"]["doc"])
+        return list_doc
+
+    def remove_training_data(self, id: str, **kwargs) -> bool:
+        if id.endswith("-sql"):
+            self.milvus_client.delete(collection_name="vannasql", ids=[id])
+            return True
+        elif id.endswith("-ddl"):
+            self.milvus_client.delete(collection_name="vannaddl", ids=[id])
+            return True
+        elif id.endswith("-doc"):
+            self.milvus_client.delete(collection_name="vannadoc", ids=[id])
+            return True
+        else:
+            return False

{vanna-0.6.0 → vanna-0.6.2}/src/vanna/qdrant/qdrant.py

@@ -39,16 +39,6 @@ class Qdrant_VectorStore(VannaBase):
         TypeError: If config["client"] is not a `qdrant_client.QdrantClient` instance
     """
 
-    documentation_collection_name = "documentation"
-    ddl_collection_name = "ddl"
-    sql_collection_name = "sql"
-
-    id_suffixes = {
-        ddl_collection_name: "ddl",
-        documentation_collection_name: "doc",
-        sql_collection_name: "sql",
-    }
-
     def __init__(
         self,
         config={},
@@ -80,15 +70,21 @@ class Qdrant_VectorStore(VannaBase):
         self.collection_params = config.get("collection_params", {})
         self.distance_metric = config.get("distance_metric", models.Distance.COSINE)
         self.documentation_collection_name = config.get(
-            "documentation_collection_name", self.documentation_collection_name
+            "documentation_collection_name", "documentation"
         )
         self.ddl_collection_name = config.get(
-            "ddl_collection_name", self.ddl_collection_name
+            "ddl_collection_name", "ddl"
         )
         self.sql_collection_name = config.get(
-            "sql_collection_name", self.sql_collection_name
+            "sql_collection_name", "sql"
         )
 
+        self.id_suffixes = {
+            self.ddl_collection_name: "ddl",
+            self.documentation_collection_name: "doc",
+            self.sql_collection_name: "sql",
+        }
+
         self._setup_collections()
 
     def add_question_sql(self, question: str, sql: str, **kwargs) -> str:
@@ -212,7 +208,7 @@ class Qdrant_VectorStore(VannaBase):
         try:
             id, collection_name = self._parse_point_id(id)
             res = self._client.delete(collection_name, points_selector=[id])
-            res == UpdateStatus.COMPLETED
+            return True
         except ValueError:
             return False