MindsDB 25.1.3.0__py3-none-any.whl → 25.1.4.0__py3-none-any.whl

mindsdb/integrations/handlers/jira_handler/jira_handler.py

@@ -55,9 +55,23 @@ class JiraHandler(APIHandler):
             return self.connection
         
         s = requests.Session()
+        if self.connection_data.get("cloud", False):
+            params = {
+                "cloud": True,
+                "username": self.connection_data['jira_username'],
+                "password": self.connection_data['jira_api_token'],
+                "url": self.connection_data['jira_url'],
+            }   
+        else:
+            params = {
+                "cloud": False,
+                "url": self.connection_data['jira_url'],
+                "session": s
+            }
+
         s.headers['Authorization'] =  f"Bearer {self.connection_data['jira_api_token']}"
 
-        self.connection = Jira(url= self.connection_data['jira_url'], session=s)
+        self.connection = Jira(**params)
         self.is_connected = True
 
 

mindsdb/integrations/handlers/jira_handler/jira_table.py

@@ -10,9 +10,26 @@ from mindsdb_sql_parser import ast
 
 logger = log.getLogger(__name__)
 
+
+def flatten_json(nested_json, parent_key="", separator="."):
+    """
+    Recursively flattens a nested JSON object into a dictionary with dot notation keys.
+    """
+    items = []
+    for k, v in nested_json.items():
+        new_key = f"{parent_key}{separator}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(flatten_json(v, new_key, separator=separator).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
 class JiraProjectsTable(APITable):
     """Jira Projects Table implementation"""
+
     _MAX_API_RESULTS = 100
+
     def select(self, query: ast.Select) -> pd.DataFrame:
         """Pulls data from the Jira "get_all_project_issues" API endpoint
         Parameters
@@ -42,8 +59,8 @@ class JiraProjectsTable(APITable):
 
             for an_order in query.order_by:
                 if an_order.field.parts[0] != "key":
-                    continue    
-                if an_order.field.parts[1] in ["reporter","assignee","status"]:
+                    continue
+                if an_order.field.parts[1] in ["reporter", "assignee", "status"]:
                     if issues_kwargs != {}:
                         raise ValueError(
                             "Duplicate order conditions found for reporter,status and assignee"
@@ -61,9 +78,9 @@ class JiraProjectsTable(APITable):
                     raise ValueError(
                         f"Order by unknown column {an_order.field.parts[1]}"
                     )
-        project = self.handler.connection_data['project']
+        project = self.handler.connection_data["project"]
         jira_project_df = self.call_jira_api(project)
-        
+
         selected_columns = []
         for target in query.targets:
             if isinstance(target, ast.Star):
@@ -74,7 +91,6 @@ class JiraProjectsTable(APITable):
             else:
                 raise ValueError(f"Unknown query target {type(target)}")
 
-
         if len(jira_project_df) == 0:
             jira_project_df = pd.DataFrame([], columns=selected_columns)
             return jira_project_df
@@ -88,7 +104,7 @@ class JiraProjectsTable(APITable):
                 by=order_by_conditions["columns"],
                 ascending=order_by_conditions["ascending"],
             )
-        
+
         if query.limit:
             jira_project_df = jira_project_df.head(total_results)
 
@@ -102,12 +118,12 @@ class JiraProjectsTable(APITable):
             List of columns
         """
         return [
-        'key',
-        'summary',
-        'status',
-        'reporter',
-        'assignee',
-        'priority',
+            "key",
+            "summary",
+            "status",
+            "reporter",
+            "assignee",
+            "priority",
         ]
 
     def call_jira_api(self, project):
@@ -116,36 +132,41 @@ class JiraProjectsTable(APITable):
         max_records = jira.get_project_issues_count(project)
         max_records = 100
         jql_query = self.handler.construct_jql()
-        max_results = self._MAX_API_RESULTS 
+        max_results = self._MAX_API_RESULTS
         start_index = 0
         total = 1
         fields = [
-        'key',
-        'fields.summary',
-        'fields.status.name',
-        'fields.reporter.name',
-        'fields.assignee.name',
-        'fields.priority.name',
+            "key",
+            "fields.summary",
+            "fields.status.name",
+            "fields.reporter.displayName",
+            "fields.assignee.displayName",
+            "fields.priority.name",
         ]
 
         all_jira_issues_df = pd.DataFrame(columns=fields)
 
         while start_index <= total:
-            results = self.handler.connect().jql(jql_query,start=start_index, limit=max_results)
-            df = pd.json_normalize(results['issues'])
+            results = self.handler.connect().jql(
+                jql_query, start=start_index, limit=max_results
+            )
+            flattened_data = [flatten_json(item) for item in results["issues"]]
+            df = pd.DataFrame(flattened_data)
             df = df[fields]
             start_index += max_results
-            total = max_records
+            total = results["total"]
             all_jira_issues_df = pd.concat([all_jira_issues_df, df], axis=0)
 
+        all_jira_issues_df = all_jira_issues_df.rename(
+            columns={
+                "key": "key",
+                "fields.summary": "summary",
+                "fields.reporter.displayName": "reporter",
+                "fields.assignee.displayName": "assignee",
+                "fields.priority.name": "priority",
+                "fields.status.name": "status",
+            },
+            errors="ignore",
+        )
 
-        all_jira_issues_df = all_jira_issues_df.rename(columns={
-                                                                'key': 'key', 
-                                                                'fields.summary': 'summary',
-                                                                'fields.reporter.name':'reporter',
-                                                                'fields.assignee.name':'assignee',
-                                                                'fields.priority.name':'priority',
-                                                                'fields.status.name':'status'})
-        
         return all_jira_issues_df
-

mindsdb/integrations/handlers/langchain_handler/requirements.txt

@@ -3,6 +3,6 @@ wikipedia==1.4.0
 tiktoken
 anthropic>=0.26.1
 litellm==1.44.8
-chromadb # Knowledge bases.
+chromadb~=0.6.3 # Knowledge bases.
 -r mindsdb/integrations/handlers/openai_handler/requirements.txt
 -r mindsdb/integrations/handlers/langchain_embedding_handler/requirements.txt

mindsdb/integrations/handlers/pinecone_handler/pinecone_handler.py

@@ -1,8 +1,10 @@
+import ast
 from typing import List, Optional
 
-import pinecone
+import numpy as np
+from pinecone import Pinecone, ServerlessSpec
+from pinecone.core.openapi.shared.exceptions import NotFoundException, PineconeApiException
 import pandas as pd
-import ast
 
 from mindsdb.integrations.libs.response import RESPONSE_TYPE
 from mindsdb.integrations.libs.response import HandlerResponse
@@ -18,32 +20,30 @@ from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
 
+DEFAULT_CREATE_TABLE_PARAMS = {
+    "dimension": 8,
+    "metric": "cosine",
+    "spec": {
+        "cloud": "aws",
+        "region": "us-east-1"
+    }
+}
+MAX_FETCH_LIMIT = 10000
+UPSERT_BATCH_SIZE = 99  # API reccomendation
+
 
 class PineconeHandler(VectorStoreHandler):
     """This handler handles connection and execution of the Pinecone statements."""
 
     name = "pinecone"
 
-    def __init__(self, name: str, **kwargs):
+    def __init__(self, name: str, connection_data: dict, **kwargs):
         super().__init__(name)
-        self.MAX_FETCH_LIMIT = 10000
-        self._connection_data = kwargs.get("connection_data")
-        self._client_config = {
-            "api_key": self._connection_data.get("api_key"),
-            "environment": self._connection_data.get("environment")
-        }
-        self._table_create_params = {
-            "dimension": 8,
-            "metric": "cosine",
-            "pods": 1,
-            "replicas": 1,
-            "pod_type": 'p1',
-        }
-        for key in self._table_create_params:
-            if key in self._connection_data:
-                self._table_create_params[key] = self._connection_data[key]
+        self.connection_data = connection_data
+        self.kwargs = kwargs
+
+        self.connection = None
         self.is_connected = False
-        self.connect()
 
     def __del__(self):
         if self.is_connected is True:
@@ -51,7 +51,8 @@ class PineconeHandler(VectorStoreHandler):
 
     def _get_index_handle(self, index_name):
         """Returns handler to index specified by `index_name`"""
-        index = pinecone.Index(index_name)
+        connection = self.connect()
+        index = connection.Index(index_name)
         try:
             index.describe_index_stats()
         except Exception:
@@ -135,10 +136,15 @@ class PineconeHandler(VectorStoreHandler):
 
     def connect(self):
         """Connect to a pinecone database."""
+        if self.is_connected is True:
+            return self.connection
+
+        if 'api_key' not in self.connection_data:
+            raise ValueError('Required parameter (api_key) must be provided.')
+
         try:
-            pinecone.init(api_key=self._client_config["api_key"], environment=self._client_config["environment"])
-            pinecone.list_indexes()
-            self.is_connected = True
+            self.connection = Pinecone(api_key=self.connection_data['api_key'])
+            return self.connection
         except Exception as e:
             logger.error(f"Error connecting to Pinecone client, {e}!")
             self.is_connected = False
@@ -147,55 +153,99 @@ class PineconeHandler(VectorStoreHandler):
         """Close the pinecone connection."""
         if self.is_connected is False:
             return
-        pinecone.init(api_key="", environment="")
+        self.connection = None
         self.is_connected = False
 
     def check_connection(self):
         """Check the connection to pinecone."""
-        response_code = StatusResponse(False)
+        response = StatusResponse(False)
+        need_to_close = self.is_connected is False
+
         try:
-            pinecone.list_indexes()
-            response_code.success = True
+            connection = self.connect()
+            connection.list_indexes()
+            response.success = True
         except Exception as e:
             logger.error(f"Error connecting to pinecone , {e}!")
-            response_code.error_message = str(e)
-        return response_code
+            response.error_message = str(e)
+
+        if response.success is True and need_to_close:
+            self.disconnect()
+        if response.success is False and self.is_connected is True:
+            self.is_connected = False
+
+        return response
 
     def get_tables(self) -> HandlerResponse:
         """Get the list of indexes in the pinecone database."""
-        indexes = pinecone.list_indexes()
-        indexes_names = pd.DataFrame(
-            columns=["index_name"],
-            data=[index for index in indexes],
+        connection = self.connect()
+        indexes = connection.list_indexes()
+        df = pd.DataFrame(
+            columns=["table_name"],
+            data=[index['name'] for index in indexes],
         )
-        return Response(resp_type=RESPONSE_TYPE.TABLE, data_frame=indexes_names)
+        return Response(resp_type=RESPONSE_TYPE.TABLE, data_frame=df)
 
     def create_table(self, table_name: str, if_not_exists=True):
         """Create an index with the given name in the Pinecone database."""
-        pinecone.create_index(name=table_name, **self._table_create_params)
+        connection = self.connect()
+
+        # TODO: Should other parameters be supported? Pod indexes?
+        # TODO: Should there be a better way to provide these parameters rather than when establishing the connection?
+        create_table_params = {}
+        for key, val in DEFAULT_CREATE_TABLE_PARAMS.items():
+            if key in self.connection_data:
+                create_table_params[key] = self.connection_data[key]
+            else:
+                create_table_params[key] = val
+
+        create_table_params["spec"] = ServerlessSpec(**create_table_params["spec"])
+
+        try:
+            connection.create_index(name=table_name, **create_table_params)
+        except PineconeApiException as pinecone_error:
+            if pinecone_error.status == 409 and if_not_exists:
+                return
+            raise Exception(f"Error creating index '{table_name}': {pinecone_error}")
 
-    def insert(self, table_name: str, data: pd.DataFrame, columns: List[str] = None):
+    def insert(self, table_name: str, data: pd.DataFrame):
         """Insert data into pinecone index passed in through `table_name` parameter."""
-        upsert_batch_size = 99  # API reccomendation
         index = self._get_index_handle(table_name)
         if index is None:
             raise Exception(f"Error getting index '{table_name}', are you sure the name is correct?")
 
         data.rename(columns={
             TableField.ID.value: "id",
-            TableField.EMBEDDINGS.value: "values",
-            TableField.METADATA.value: "metadata"},
+            TableField.EMBEDDINGS.value: "values"},
             inplace=True)
-        data = data[["id", "values", "metadata"]]
 
-        for chunk in (data[pos:pos + upsert_batch_size] for pos in range(0, len(data), upsert_batch_size)):
+        columns = ["id", "values"]
+
+        if TableField.METADATA.value in data.columns:
+            data.rename(columns={TableField.METADATA.value: "metadata"}, inplace=True)
+            # fill None and NaN values with empty dict
+            if data['metadata'].isnull().any():
+                data['metadata'] = data['metadata'].apply(lambda x: {} if x is None or (isinstance(x, float) and np.isnan(x)) else x)
+            columns.append("metadata")
+
+        data = data[columns]
+
+        # convert the embeddings to lists if they are strings
+        data["values"] = data["values"].apply(lambda x: ast.literal_eval(x) if isinstance(x, str) else x)
+
+        for chunk in (data[pos:pos + UPSERT_BATCH_SIZE] for pos in range(0, len(data), UPSERT_BATCH_SIZE)):
             chunk = chunk.to_dict(orient="records")
             index.upsert(vectors=chunk)
 
     def drop_table(self, table_name: str, if_exists=True):
         """Delete an index passed in through `table_name` from the pinecone ."""
-
-        pinecone.delete_index(table_name)
+        connection = self.connect()
+        try:
+            connection.delete_index(table_name)
+        except NotFoundException:
+            if if_exists:
+                return
+            raise Exception(f"Error deleting index '{table_name}', are you sure the name is correct?")
 
     def delete(self, table_name: str, conditions: List[FilterCondition] = None):
         """Delete records in pinecone index `table_name` based on ids or based on metadata conditions."""
@@ -225,6 +275,7 @@ class PineconeHandler(VectorStoreHandler):
         limit: int = None,
     ):
         """Run query on pinecone index named `table_name` and get results."""
+        # TODO: Add support for namespaces.
         index = self._get_index_handle(table_name)
         if index is None:
             raise Exception(f"Error getting index '{table_name}', are you sure the name is correct?")
@@ -233,23 +284,28 @@ class PineconeHandler(VectorStoreHandler):
             "include_values": True,
             "include_metadata": True
         }
+
         # check for metadata filter
         metadata_filters = self._translate_metadata_condition(conditions)
-        # check for vector filter
-        vector_filter = (
-            None
-            if conditions is None
-            else [
-                condition.value
-                for condition in conditions
-                if condition.column == TableField.SEARCH_VECTOR.value
-            ]
-        )
-        if vector_filter:
-            if len(vector_filter) > 1:
+        if metadata_filters is not None:
+            query["filter"] = metadata_filters
+
+        # check for vector and id filters
+        vector_filters = []
+        id_filters = []
+
+        if conditions:
+            for condition in conditions:
+                if condition.column == TableField.SEARCH_VECTOR.value:
+                    vector_filters.append(condition.value)
+                elif condition.column == TableField.ID.value:
+                    id_filters.append(condition.value)
+
+        if vector_filters:
+            if len(vector_filters) > 1:
                 raise Exception("You cannot have multiple search_vectors in query")
 
-            query["vector"] = vector_filter[0]
+            query["vector"] = vector_filters[0]
             # For subqueries, the vector filter is a list of list of strings
             if isinstance(query["vector"], list) and isinstance(query["vector"][0], str):
                 if len(query["vector"]) > 1:
@@ -260,26 +316,21 @@ class PineconeHandler(VectorStoreHandler):
                 except Exception as e:
                     raise Exception(f"Cannot parse the search vector '{query['vector']}'into a list: {e}")
 
-        # check for limit
-        if limit is not None:
-            query["top_k"] = limit
-        else:
-            query["top_k"] = self.MAX_FETCH_LIMIT
-        if metadata_filters is not None:
-            query["filter"] = metadata_filters
-        # check for id filter
-        id_filters = None
-        if conditions is not None:
-            id_filters = [
-                condition.value
-                for condition in conditions
-                if condition.column == TableField.ID.value
-            ] or None
         if id_filters:
             if len(id_filters) > 1:
                 raise Exception("You cannot have multiple IDs in query")
 
             query["id"] = id_filters[0]
+
+        if not vector_filters and not id_filters:
+            raise Exception("You must provide either a search_vector or an ID in the query")
+
+        # check for limit
+        if limit is not None:
+            query["top_k"] = limit
+        else:
+            query["top_k"] = MAX_FETCH_LIMIT
+
         # exec query
         try:
             result = index.query(**query)

mindsdb/integrations/handlers/pinecone_handler/requirements.txt

@@ -1 +1 @@
-pinecone-client
+pinecone-client==5.0.1

mindsdb/integrations/handlers/postgres_handler/postgres_handler.py

@@ -1,5 +1,6 @@
 import time
 import json
+from typing import Optional
 
 import pandas as pd
 import psycopg
@@ -281,21 +282,27 @@ class PostgresHandler(DatabaseHandler):
         """
         return self.native_query(query)
 
-    def get_columns(self, table_name: str) -> Response:
+    def get_columns(self, table_name: str, schema_name: Optional[str] = None) -> Response:
         """
         Retrieves column details for a specified table in the PostgreSQL database.
 
         Args:
             table_name (str): The name of the table for which to retrieve column information.
+            schema_name (str): The name of the schema in which the table is located.
 
         Returns:
             Response: A response object containing the column details, formatted as per the `Response` class.
+
         Raises:
             ValueError: If the 'table_name' is not a valid string.
         """
 
         if not table_name or not isinstance(table_name, str):
             raise ValueError("Invalid table name provided.")
+        if isinstance(schema_name, str):
+            schema_name = f"'{schema_name}'"
+        else:
+            schema_name = 'current_schema()'
         query = f"""
             SELECT
                 column_name as "Field",
@@ -305,12 +312,11 @@ class PostgresHandler(DatabaseHandler):
             WHERE
                 table_name = '{table_name}'
             AND
-                table_schema = current_schema()
+                table_schema = {schema_name}
         """
         return self.native_query(query)
 
     def subscribe(self, stop_event, callback, table_name, columns=None, **kwargs):
-
         config = self._make_connection_args()
         config['autocommit'] = True
 

mindsdb/integrations/handlers/slack_handler/slack_handler.py

@@ -231,6 +231,9 @@ class SlackHandler(APIChatHandler):
             'polling': {
                 'type': 'realtime',
             },
+            'memory': {
+                'type': 'handler',
+            },
             'tables': [
                 {
                     'chat_table': {
@@ -238,7 +241,7 @@ class SlackHandler(APIChatHandler):
                         'chat_id_col': 'channel_id',
                         'username_col': 'user',
                         'text_col': 'text',
-                        'time_col': 'thread_ts',
+                        'time_col': 'created_at',
                     }
                 },
                 {
@@ -264,7 +267,7 @@ class SlackHandler(APIChatHandler):
         user_info = web_connection.auth_test().data
         return user_info['bot_id']
 
-    def subscribe(self, stop_event: threading.Event, callback: Callable, **kwargs: Any) -> None:
+    def subscribe(self, stop_event: threading.Event, callback: Callable, table_name: Text, columns: List = None, **kwargs: Any) -> None:
         """
         Subscribes to the Slack API using the Socket Mode for real-time responses to messages.
 
@@ -274,6 +277,14 @@ class SlackHandler(APIChatHandler):
             table_name (Text): The name of the table to subscribe to.
             kwargs: Arbitrary keyword arguments.
         """
+        if table_name not in ['messages', 'threads']:
+            raise RuntimeError(f'Table {table_name} is not supported for subscription.')
+
+        # Raise an error if columns are provided.
+        # Since Slack subscriptions depend on events and not changes to the virtual tables, columns are not supported.
+        if columns:
+            raise RuntimeError('Columns are not supported for Slack subscriptions.')
+
         self._socket_connection = SocketModeClient(
             # This app-level token will be used only for establishing a connection.
             app_token=self.connection_data['app_token'],  # xapp-A111-222-xyz

mindsdb/integrations/handlers/slack_handler/slack_tables.py

@@ -6,7 +6,7 @@ import pandas as pd
 from slack_sdk.errors import SlackApiError
 
 from mindsdb.integrations.libs.api_handler import APIResource
-from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions, FilterCondition, FilterOperator
+from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions, FilterCondition, FilterOperator, SortColumn
 from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
@@ -203,6 +203,7 @@ class SlackMessagesTable(APIResource):
         self,
         conditions: List[FilterCondition] = None,
         limit: int = None,
+        sort: List[SortColumn] = None,
         **kwargs: Any
     ) -> pd.DataFrame:
         """
@@ -222,6 +223,7 @@ class SlackMessagesTable(APIResource):
         Args:
             conditions (List[FilterCondition]): The conditions to filter the messages.
             limit (int): The limit of the messages to return.
+            sort (List[SortColumn]): The columns to sort the messages by.
             kwargs (Any): Arbitrary keyword arguments.
 
         Raises:
@@ -306,6 +308,14 @@ class SlackMessagesTable(APIResource):
         # Translate the time stamp into a 'created_at' field.
         result['created_at'] = pd.to_datetime(result['ts'].astype(float), unit='s').dt.strftime('%Y-%m-%d %H:%M:%S')
 
+        # Sort the messages by the specified columns.
+        if sort:
+            result.sort_values(
+                by=[col.column for col in sort],
+                ascending=[col.ascending for col in sort],
+                inplace=True
+            )
+
         return result
 
     def insert(self, query: Insert):
@@ -496,6 +506,7 @@ class SlackThreadsTable(APIResource):
         self,
         conditions: List[FilterCondition] = None,
         limit: int = None,
+        sort: List[SortColumn] = None,
         **kwargs: Any
     ) -> pd.DataFrame:
         """
@@ -514,6 +525,7 @@ class SlackThreadsTable(APIResource):
         Args:
             conditions (List[FilterCondition]): The conditions to filter the messages.
             limit (int): The limit of the messages to return.
+            sort (List[SortColumn]): The columns to sort the messages by.
             kwargs (Any): Arbitrary keyword arguments.
 
         Raises:
@@ -591,6 +603,14 @@ class SlackThreadsTable(APIResource):
         result['channel_id'] = params['channel']
         result['channel_name'] = channel['name'] if 'name' in channel else None
 
+        # Sort the messages by the specified columns.
+        if sort:
+            result.sort_values(
+                by=[col.column for col in sort],
+                ascending=[col.ascending for col in sort],
+                inplace=True
+            )
+
         return result
 
     def insert(self, query: Insert):

mindsdb/integrations/libs/ml_handler_process/learn_process.py

@@ -8,7 +8,7 @@ from sqlalchemy.orm.attributes import flag_modified
 from mindsdb_sql_parser import parse_sql
 from mindsdb_sql_parser.ast import Identifier, Select, Star, NativeQuery
 
-from mindsdb.api.executor import SQLQuery
+from mindsdb.api.executor.sql_query import SQLQuery
 import mindsdb.utilities.profiler as profiler
 from mindsdb.utilities.functions import mark_process
 from mindsdb.utilities.config import Config

mindsdb/integrations/utilities/rag/loaders/vector_store_loader/pgvector.py

@@ -72,7 +72,7 @@ class PGVectorMDB(PGVector):
                 # Use inner product for sparse vectors
                 distance_op = "<#>"
                 # For inner product, larger values are better matches
-                order_direction = "DESC"
+                order_direction = "ASC"
             else:
                 # Dense vectors: expect string in JSON array format or list of floats
                 if isinstance(embedding, list):