MindsDB 25.1.2.0__py3-none-any.whl → 25.1.5.0__py3-none-any.whl

mindsdb/integrations/handlers/jira_handler/jira_table.py

@@ -10,9 +10,26 @@ from mindsdb_sql_parser import ast
 
 logger = log.getLogger(__name__)
 
+
+def flatten_json(nested_json, parent_key="", separator="."):
+    """
+    Recursively flattens a nested JSON object into a dictionary with dot notation keys.
+    """
+    items = []
+    for k, v in nested_json.items():
+        new_key = f"{parent_key}{separator}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(flatten_json(v, new_key, separator=separator).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
 class JiraProjectsTable(APITable):
     """Jira Projects Table implementation"""
+
     _MAX_API_RESULTS = 100
+
     def select(self, query: ast.Select) -> pd.DataFrame:
         """Pulls data from the Jira "get_all_project_issues" API endpoint
         Parameters
@@ -42,8 +59,8 @@ class JiraProjectsTable(APITable):
 
             for an_order in query.order_by:
                 if an_order.field.parts[0] != "key":
-                    continue    
-                if an_order.field.parts[1] in ["reporter","assignee","status"]:
+                    continue
+                if an_order.field.parts[1] in ["reporter", "assignee", "status"]:
                     if issues_kwargs != {}:
                         raise ValueError(
                             "Duplicate order conditions found for reporter,status and assignee"
@@ -61,9 +78,9 @@ class JiraProjectsTable(APITable):
                     raise ValueError(
                         f"Order by unknown column {an_order.field.parts[1]}"
                     )
-        project = self.handler.connection_data['project']
+        project = self.handler.connection_data["project"]
         jira_project_df = self.call_jira_api(project)
-        
+
         selected_columns = []
         for target in query.targets:
             if isinstance(target, ast.Star):
@@ -74,7 +91,6 @@ class JiraProjectsTable(APITable):
             else:
                 raise ValueError(f"Unknown query target {type(target)}")
 
-
         if len(jira_project_df) == 0:
             jira_project_df = pd.DataFrame([], columns=selected_columns)
             return jira_project_df
@@ -88,7 +104,7 @@ class JiraProjectsTable(APITable):
                 by=order_by_conditions["columns"],
                 ascending=order_by_conditions["ascending"],
             )
-        
+
         if query.limit:
             jira_project_df = jira_project_df.head(total_results)
 
@@ -102,12 +118,12 @@ class JiraProjectsTable(APITable):
             List of columns
         """
         return [
-        'key',
-        'summary',
-        'status',
-        'reporter',
-        'assignee',
-        'priority',
+            "key",
+            "summary",
+            "status",
+            "reporter",
+            "assignee",
+            "priority",
         ]
 
     def call_jira_api(self, project):
@@ -116,36 +132,41 @@ class JiraProjectsTable(APITable):
         max_records = jira.get_project_issues_count(project)
         max_records = 100
         jql_query = self.handler.construct_jql()
-        max_results = self._MAX_API_RESULTS 
+        max_results = self._MAX_API_RESULTS
         start_index = 0
         total = 1
         fields = [
-        'key',
-        'fields.summary',
-        'fields.status.name',
-        'fields.reporter.name',
-        'fields.assignee.name',
-        'fields.priority.name',
+            "key",
+            "fields.summary",
+            "fields.status.name",
+            "fields.reporter.displayName",
+            "fields.assignee.displayName",
+            "fields.priority.name",
         ]
 
         all_jira_issues_df = pd.DataFrame(columns=fields)
 
         while start_index <= total:
-            results = self.handler.connect().jql(jql_query,start=start_index, limit=max_results)
-            df = pd.json_normalize(results['issues'])
+            results = self.handler.connect().jql(
+                jql_query, start=start_index, limit=max_results
+            )
+            flattened_data = [flatten_json(item) for item in results["issues"]]
+            df = pd.DataFrame(flattened_data)
             df = df[fields]
             start_index += max_results
-            total = max_records
+            total = results["total"]
             all_jira_issues_df = pd.concat([all_jira_issues_df, df], axis=0)
 
+        all_jira_issues_df = all_jira_issues_df.rename(
+            columns={
+                "key": "key",
+                "fields.summary": "summary",
+                "fields.reporter.displayName": "reporter",
+                "fields.assignee.displayName": "assignee",
+                "fields.priority.name": "priority",
+                "fields.status.name": "status",
+            },
+            errors="ignore",
+        )
 
-        all_jira_issues_df = all_jira_issues_df.rename(columns={
-                                                                'key': 'key', 
-                                                                'fields.summary': 'summary',
-                                                                'fields.reporter.name':'reporter',
-                                                                'fields.assignee.name':'assignee',
-                                                                'fields.priority.name':'priority',
-                                                                'fields.status.name':'status'})
-        
         return all_jira_issues_df
-

mindsdb/integrations/handlers/langchain_embedding_handler/fastapi_embeddings.py

@@ -0,0 +1,82 @@
+from typing import Any, List
+from langchain_core.embeddings import Embeddings
+import requests
+
+
+class FastAPIEmbeddings(Embeddings):
+    """An embedding extension that interfaces with FAST API. Useful for custom serving solutions."""
+
+    def __init__(
+        self,
+        api_base: str,
+        model: str,
+        batch_size: int = 32,
+        **kwargs: Any,
+    ):
+        """Initialize the embeddings class.
+
+        Args:
+            api_base: Base URL for the VLLM server
+            model: Model name/path to use for embeddings
+            batch_size: Batch size for generating embeddings
+        """
+        super().__init__()
+        self.api_base = api_base
+        self.model = model
+        self.batch_size = batch_size
+
+        # initialize requests here with the api_base
+
+    def _get_embeddings(self, texts: List[str]) -> List[str]:
+        """Get embeddings for a batch of text chunks.
+
+        Returns:
+            List of embeddings as strings. For sparse vectors, returns strings in format
+            "{key:value,...}/size" where size is the dimension of the vector space.
+        """
+
+        headers = {"accept": "application/json", "Content-Type": "application/json"}
+
+        data = {
+            "input": texts,
+            "model": self.model
+        }
+
+        response = requests.post(self.api_base, headers=headers, json=data)
+
+        response.raise_for_status()
+
+        embeddings = []
+        for response_dict in response.json()["data"]:
+            embedding = response_dict["embedding"]
+            embeddings.append(embedding)
+
+        return embeddings
+
+    def embed_documents(self, texts: List[str]) -> List[str]:
+        """Embed a list of documents using vLLM.
+
+        Args:
+            texts: List of documents to embed
+
+        Returns:
+            List of embeddings as strings, one for each document.
+            For sparse embeddings, returns strings in format "{key:value,...}/size"
+            For dense embeddings, returns JSON strings of float lists
+        """
+
+        return self._get_embeddings(texts)
+
+    def embed_query(self, text: str) -> str:
+        """Embed a single query text using vLLM.
+
+        Args:
+            text: Query text to embed
+
+        Returns:
+            Query embedding as a string.
+            For sparse embeddings, returns string in format "{key:value,...}/size"
+            For dense embeddings, returns JSON string of float list
+        """
+
+        return self._get_embeddings([text])[0]

mindsdb/integrations/handlers/langchain_embedding_handler/langchain_embedding_handler.py

@@ -10,6 +10,7 @@ from mindsdb.integrations.libs.base import BaseMLEngine
 from mindsdb.utilities import log
 from langchain_core.embeddings import Embeddings
 from mindsdb.integrations.handlers.langchain_embedding_handler.vllm_embeddings import VLLMEmbeddings
+from mindsdb.integrations.handlers.langchain_embedding_handler.fastapi_embeddings import FastAPIEmbeddings
 
 logger = log.getLogger(__name__)
 
@@ -20,7 +21,10 @@ logger = log.getLogger(__name__)
 # This is used for the user to select the embedding model
 EMBEDDING_MODELS = {
     'VLLM': 'VLLMEmbeddings',
-    'vllm': 'VLLMEmbeddings'
+    'vllm': 'VLLMEmbeddings',
+    'FastAPI': 'FastAPIEmbeddings',
+    'fastapi': 'FastAPIEmbeddings'
+
 }
 
 try:
@@ -55,6 +59,9 @@ def get_langchain_class(class_name: str) -> Embeddings:
     if class_name == "VLLMEmbeddings":
         return VLLMEmbeddings
 
+    if class_name == "FastAPIEmbeddings":
+        return FastAPIEmbeddings
+
     # Then try langchain_community.embeddings
     try:
         module = importlib.import_module("langchain_community.embeddings")

mindsdb/integrations/handlers/langchain_handler/requirements.txt

@@ -3,6 +3,6 @@ wikipedia==1.4.0
 tiktoken
 anthropic>=0.26.1
 litellm==1.44.8
-chromadb # Knowledge bases.
+chromadb~=0.6.3 # Knowledge bases.
 -r mindsdb/integrations/handlers/openai_handler/requirements.txt
 -r mindsdb/integrations/handlers/langchain_embedding_handler/requirements.txt

mindsdb/integrations/handlers/ms_one_drive_handler/ms_one_drive_handler.py

@@ -28,7 +28,7 @@ class MSOneDriveHandler(APIHandler):
     """
 
     name = 'one_drive'
-    supported_file_formats = ['csv', 'tsv', 'json', 'parquet']
+    supported_file_formats = ['csv', 'tsv', 'json', 'parquet', 'pdf', 'txt']
 
     def __init__(self, name: Text, connection_data: Dict, **kwargs: Any) -> None:
         """

mindsdb/integrations/handlers/ms_one_drive_handler/ms_one_drive_tables.py

@@ -9,6 +9,8 @@ from mindsdb.integrations.utilities.sql_utils import (
     SortColumn
 )
 
+from mindsdb.integrations.utilities.files.file_reader import FileReader
+
 
 class ListFilesTable(APIResource):
     """
@@ -97,4 +99,10 @@ class FileTable(APIResource):
         elif file_extension == "parquet":
             df = pd.read_parquet(BytesIO(file_content))
 
+        elif file_extension == "pdf":
+            df = FileReader().read_pdf(BytesIO(file_content))
+
+        elif file_extension == "txt":
+            df = FileReader().read_txt(BytesIO(file_content))
+
         return df

mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py

@@ -37,6 +37,11 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
         super().__init__(name=name, **kwargs)
         self._is_shared_db = False
         self._is_vector_registered = False
+        # we get these from the connection args on PostgresHandler parent
+        self._is_sparse = self.connection_args.get('is_sparse', False)
+        self._vector_size = self.connection_args.get('vector_size', None) 
+        if self._is_sparse and not self._vector_size:
+            raise ValueError("vector_size is required when is_sparse=True")
         self.connect()
 
     def _make_connection_args(self):
@@ -190,13 +195,30 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
         if filter_conditions:
 
             if embedding_search:
-                # if search vector, return similar rows, apply other filters after if any
                 search_vector = filter_conditions["embeddings"]["value"][0]
                 filter_conditions.pop("embeddings")
-                return f"SELECT {targets} FROM {table_name} ORDER BY embeddings <=> '{search_vector}' {after_from_clause}"
+
+                if self._is_sparse:
+                    # Convert dict to sparse vector if needed
+                    if isinstance(search_vector, dict):
+                        from pgvector.utils import SparseVector
+                        embedding = SparseVector(search_vector, self._vector_size)
+                        search_vector = embedding.to_text()
+                    # Use inner product for sparse vectors
+                    distance_op = "<#>"
+                else:
+                    # Convert list to vector string if needed
+                    if isinstance(search_vector, list):
+                        search_vector = f"[{','.join(str(x) for x in search_vector)}]"
+                    # Use cosine similarity for dense vectors
+                    distance_op = "<=>"
+
+                return f"SELECT {targets} FROM {table_name} ORDER BY embeddings {distance_op} '{search_vector}' ASC {after_from_clause}"
+
             else:
-                # if filter conditions, return filtered rows
+                # if filter conditions, return rows that satisfy the conditions
                 return f"SELECT {targets} FROM {table_name} {after_from_clause}"
+
         else:
             # if no filter conditions, return all rows
             return f"SELECT {targets} FROM {table_name} {after_from_clause}"
@@ -339,14 +361,30 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
         full_search_query = f'{semantic_search_cte}{full_text_search_cte}{hybrid_select}'
         return self.raw_query(full_search_query)
 
-    def create_table(self, table_name: str, if_not_exists=True):
-        """
-        Run a create table query on the pgvector database.
-        """
-        table_name = self._check_table(table_name)
-
-        query = f"CREATE TABLE IF NOT EXISTS {table_name} (id text PRIMARY KEY, content text, embeddings vector, metadata jsonb)"
-        self.raw_query(query)
+    def create_table(self, table_name: str):
+        """Create a table with a vector column."""
+        with self.connection.cursor() as cur:
+            # For sparse vectors, use sparsevec type
+            vector_column_type = 'sparsevec' if self._is_sparse else 'vector'
+            
+            # Vector size is required for sparse vectors, optional for dense
+            if self._is_sparse and not self._vector_size:
+                raise ValueError("vector_size is required for sparse vectors")
+            
+            # Add vector size specification only if provided
+            size_spec = f"({self._vector_size})" if self._vector_size is not None else "()"
+            if vector_column_type == 'vector':
+                size_spec = ''
+
+            cur.execute(f"""
+                CREATE TABLE IF NOT EXISTS {table_name} (
+                    id TEXT PRIMARY KEY,
+                    embeddings {vector_column_type}{size_spec},
+                    content TEXT,
+                    metadata JSONB
+                )
+            """)
+            self.connection.commit()
 
     def insert(
         self, table_name: str, data: pd.DataFrame
@@ -444,4 +482,3 @@ class PgVectorHandler(VectorStoreHandler, PostgresHandler):
         """
         table_name = self._check_table(table_name)
         self.raw_query(f"DROP TABLE IF EXISTS {table_name}")
-