MindsDB 25.7.1.0__py3-none-any.whl → 25.7.3.0__py3-none-any.whl

mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py

@@ -18,6 +18,8 @@ from mindsdb.integrations.libs.vectordatabase_handler import (
     DistanceFunction,
     TableField,
 )
+from mindsdb.integrations.libs.keyword_search_base import KeywordSearchBase
+from mindsdb.integrations.utilities.sql_utils import KeywordSearchArgs
 from mindsdb.utilities import log
 from mindsdb.utilities.profiler import profiler
 from mindsdb.utilities.context import context as ctx
@@ -26,7 +28,7 @@ logger = log.getLogger(__name__)
 
 
 # todo Issue #7316 add support for different indexes and search algorithms e.g. cosine similarity or L2 norm
-class PgVectorHandler(PostgresHandler, VectorStoreHandler):
+class PgVectorHandler(PostgresHandler, VectorStoreHandler, KeywordSearchBase):
     """This handler handles connection and execution of the PostgreSQL with pgvector extension statements."""
 
     name = "pgvector"
@@ -228,6 +230,40 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
         else:
             return ""
 
+    @staticmethod
+    def _construct_where_clause_with_keywords(filter_conditions=None, keyword_query=None, content_column_name=None):
+        if not keyword_query or not content_column_name:
+            return PgVectorHandler._construct_where_clause(filter_conditions)
+
+        keyword_query_condition = (
+            f"""to_tsvector('english', {content_column_name}) @@ websearch_to_tsquery('english', '{keyword_query}')"""
+        )
+        if filter_conditions is None:
+            return ""
+
+        where_clauses = []
+
+        for item in filter_conditions:
+            key = item["name"]
+
+            if item["op"].lower() in ("in", "not in"):
+                values = list(repr(i) for i in item["value"])
+                item["value"] = "({})".format(", ".join(values))
+            else:
+                if item["value"] is None:
+                    item["value"] = "null"
+                else:
+                    item["value"] = repr(item["value"])
+            where_clauses.append(f"{key} {item['op']} {item['value']}")
+
+        where_clauses.append(keyword_query_condition)
+        if len(where_clauses) > 1:
+            return f"WHERE {' AND '.join(where_clauses)}"
+        elif len(where_clauses) == 1:
+            return f"WHERE {where_clauses[0]}"
+        else:
+            return ""
+
     @staticmethod
     def _construct_full_after_from_clause(
         where_clause: str,
@@ -236,6 +272,36 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
     ) -> str:
         return f"{where_clause} {offset_clause} {limit_clause}"
 
+    def _build_keyword_bm25_query(
+        self,
+        table_name: str,
+        query: str,
+        columns: List[str] = None,
+        content_column_name: str = "content",
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        offset: int = None,
+    ):
+        if columns is None:
+            columns = ["id", "content", "metadata"]
+
+        filter_conditions, _ = self._translate_conditions(conditions)
+
+        # given filter conditions, construct where clause
+        where_clause = self._construct_where_clause_with_keywords(filter_conditions, query, content_column_name)
+
+        query = f"""
+            SELECT
+                {", ".join(columns)},
+                ts_rank_cd(to_tsvector('english', {content_column_name}), websearch_to_tsquery('english', '{query}')) as distance
+            FROM
+                {table_name}
+            {where_clause if where_clause else ""}
+            {f"LIMIT {limit}" if limit else ""}
+            {f"OFFSET {offset}" if offset else ""};"""
+
+        return query
+
     def _build_select_query(
         self,
         table_name: str,
@@ -320,6 +386,33 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
             columns = ["id", "content", "embeddings", "metadata"]
 
         query = self._build_select_query(table_name, columns, conditions, limit, offset)
+
+        result = self.raw_query(query)
+
+        # ensure embeddings are returned as string so they can be parsed by mindsdb
+        if "embeddings" in columns:
+            result["embeddings"] = result["embeddings"].astype(str)
+
+        return result
+
+    def keyword_select(
+        self,
+        table_name: str,
+        columns: List[str] = None,
+        conditions: List[FilterCondition] = None,
+        offset: int = None,
+        limit: int = None,
+        keyword_search_args: KeywordSearchArgs = None,
+    ) -> pd.DataFrame:
+        table_name = self._check_table(table_name)
+
+        if columns is None:
+            columns = ["id", "content", "embeddings", "metadata"]
+        content_column_name = keyword_search_args.column
+        query = self._build_keyword_bm25_query(
+            table_name, keyword_search_args.query, columns, content_column_name, conditions, limit, offset
+        )
+
         result = self.raw_query(query)
 
         # ensure embeddings are returned as string so they can be parsed by mindsdb

mindsdb/integrations/handlers/s3_handler/s3_handler.py

@@ -7,6 +7,7 @@ from duckdb import HTTPException
 from mindsdb_sql_parser import parse_sql
 import pandas as pd
 from typing import Text, Dict, Optional
+from botocore.client import Config
 from botocore.exceptions import ClientError
 
 from mindsdb_sql_parser.ast.base import ASTNode
@@ -16,7 +17,7 @@ from mindsdb.utilities import log
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
     HandlerResponse as Response,
-    RESPONSE_TYPE
+    RESPONSE_TYPE,
 )
 
 from mindsdb.integrations.libs.api_handler import APIResource, APIHandler
@@ -26,16 +27,12 @@ logger = log.getLogger(__name__)
 
 
 class ListFilesTable(APIResource):
-
-    def list(self,
-             targets: List[str] = None,
-             conditions: List[FilterCondition] = None,
-             limit: int = None,
-             *args, **kwargs) -> pd.DataFrame:
-
+    def list(
+        self, targets: List[str] = None, conditions: List[FilterCondition] = None, limit: int = None, *args, **kwargs
+    ) -> pd.DataFrame:
         buckets = None
         for condition in conditions:
-            if condition.column == 'bucket':
+            if condition.column == "bucket":
                 if condition.op == FilterOperator.IN:
                     buckets = condition.value
                 elif condition.op == FilterOperator.EQUAL:
@@ -44,25 +41,27 @@ class ListFilesTable(APIResource):
 
         data = []
         for obj in self.handler.get_objects(limit=limit, buckets=buckets):
-            path = obj['Key']
-            path = path.replace('`', '')
+            path = obj["Key"]
+            path = path.replace("`", "")
             item = {
-                'path': path,
-                'bucket': obj['Bucket'],
-                'name': path[path.rfind('/') + 1:],
-                'extension': path[path.rfind('.') + 1:]
+                "path": path,
+                "bucket": obj["Bucket"],
+                "name": path[path.rfind("/") + 1 :],
+                "extension": path[path.rfind(".") + 1 :],
             }
 
+            if targets and "public_url" in targets:
+                item["public_url"] = self.handler.generate_sas_url(path, obj["Bucket"])
+
             data.append(item)
 
         return pd.DataFrame(data=data, columns=self.get_columns())
 
     def get_columns(self) -> List[str]:
-        return ["path", "name", "extension", "bucket", "content"]
+        return ["path", "name", "extension", "bucket", "content", "public_url"]
 
 
 class FileTable(APIResource):
-
     def list(self, targets: List[str] = None, table_name=None, *args, **kwargs) -> pd.DataFrame:
         return self.handler.read_as_table(table_name)
 
@@ -76,9 +75,9 @@ class S3Handler(APIHandler):
     This handler handles connection and execution of the SQL statements on AWS S3.
     """
 
-    name = 's3'
+    name = "s3"
     # TODO: Can other file formats be supported?
-    supported_file_formats = ['csv', 'tsv', 'json', 'parquet']
+    supported_file_formats = ["csv", "tsv", "json", "parquet"]
 
     def __init__(self, name: Text, connection_data: Optional[Dict], **kwargs):
         """
@@ -96,7 +95,7 @@ class S3Handler(APIHandler):
         self.connection = None
         self.is_connected = False
         self.thread_safe = True
-        self.bucket = self.connection_data.get('bucket')
+        self.bucket = self.connection_data.get("bucket")
         self._regions = {}
 
         self._files_table = ListFilesTable(self)
@@ -119,8 +118,8 @@ class S3Handler(APIHandler):
             return self.connection
 
         # Validate mandatory parameters.
-        if not all(key in self.connection_data for key in ['aws_access_key_id', 'aws_secret_access_key']):
-            raise ValueError('Required parameters (aws_access_key_id, aws_secret_access_key) must be provided.')
+        if not all(key in self.connection_data for key in ["aws_access_key_id", "aws_secret_access_key"]):
+            raise ValueError("Required parameters (aws_access_key_id, aws_secret_access_key) must be provided.")
 
         # Connect to S3 and configure mandatory credentials.
         self.connection = self._connect_boto3()
@@ -152,13 +151,13 @@ class S3Handler(APIHandler):
         duckdb_conn.execute(f"SET s3_secret_access_key='{self.connection_data['aws_secret_access_key']}'")
 
         # Configure optional parameters.
-        if 'aws_session_token' in self.connection_data:
+        if "aws_session_token" in self.connection_data:
             duckdb_conn.execute(f"SET s3_session_token='{self.connection_data['aws_session_token']}'")
 
         # detect region for bucket
         if bucket not in self._regions:
             client = self.connect()
-            self._regions[bucket] = client.get_bucket_location(Bucket=bucket)['LocationConstraint']
+            self._regions[bucket] = client.get_bucket_location(Bucket=bucket)["LocationConstraint"]
 
         region = self._regions[bucket]
         duckdb_conn.execute(f"SET s3_region='{region}'")
@@ -177,15 +176,17 @@ class S3Handler(APIHandler):
         """
         # Configure mandatory credentials.
         config = {
-            'aws_access_key_id': self.connection_data['aws_access_key_id'],
-            'aws_secret_access_key': self.connection_data['aws_secret_access_key']
+            "aws_access_key_id": self.connection_data["aws_access_key_id"],
+            "aws_secret_access_key": self.connection_data["aws_secret_access_key"],
         }
 
         # Configure optional parameters.
-        if 'aws_session_token' in self.connection_data:
-            config['aws_session_token'] = self.connection_data['aws_session_token']
+        optional_parameters = ["region_name", "aws_session_token"]
+        for parameter in optional_parameters:
+            if parameter in self.connection_data:
+                config[parameter] = self.connection_data[parameter]
 
-        client = boto3.client('s3', **config)
+        client = boto3.client("s3", **config, config=Config(signature_version="s3v4"))
 
         # check connection
         if self.bucket is not None:
@@ -219,7 +220,7 @@ class S3Handler(APIHandler):
             self._connect_boto3()
             response.success = True
         except (ClientError, ValueError) as e:
-            logger.error(f'Error connecting to S3 with the given credentials, {e}!')
+            logger.error(f"Error connecting to S3 with the given credentials, {e}!")
             response.error_message = str(e)
 
         if response.success and need_to_close:
@@ -235,8 +236,8 @@ class S3Handler(APIHandler):
             return self.bucket, key
 
         # get bucket from first part of the key
-        ar = key.split('/')
-        return ar[0], '/'.join(ar[1:])
+        ar = key.split("/")
+        return ar[0], "/".join(ar[1:])
 
     def read_as_table(self, key) -> pd.DataFrame:
         """
@@ -245,7 +246,6 @@ class S3Handler(APIHandler):
         bucket, key = self._get_bucket(key)
 
         with self._connect_duckdb(bucket) as connection:
-
             cursor = connection.execute(f"SELECT * FROM 's3://{bucket}/{key}'")
 
             return cursor.fetchdf()
@@ -259,7 +259,7 @@ class S3Handler(APIHandler):
         client = self.connect()
 
         obj = client.get_object(Bucket=bucket, Key=key)
-        content = obj['Body'].read()
+        content = obj["Body"].read()
         return content
 
     def add_data_to_table(self, key, df) -> None:
@@ -277,7 +277,7 @@ class S3Handler(APIHandler):
             client = self.connect()
             client.head_object(Bucket=bucket, Key=key)
         except ClientError as e:
-            logger.error(f'Error querying the file {key} in the bucket {bucket}, {e}!')
+            logger.error(f"Error querying the file {key} in the bucket {bucket}, {e}!")
             raise e
 
         with self._connect_duckdb(bucket) as connection:
@@ -309,31 +309,28 @@ class S3Handler(APIHandler):
         if isinstance(query, Select):
             table_name = query.from_table.parts[-1]
 
-            if table_name == 'files':
+            if table_name == "files":
                 table = self._files_table
                 df = table.select(query)
 
                 # add content
                 has_content = False
                 for target in query.targets:
-                    if isinstance(target, Identifier) and target.parts[-1].lower() == 'content':
+                    if isinstance(target, Identifier) and target.parts[-1].lower() == "content":
                         has_content = True
                         break
                 if has_content:
-                    df['content'] = df['path'].apply(self._read_as_content)
+                    df["content"] = df["path"].apply(self._read_as_content)
             else:
-                extension = table_name.split('.')[-1]
+                extension = table_name.split(".")[-1]
                 if extension not in self.supported_file_formats:
-                    logger.error(f'The file format {extension} is not supported!')
-                    raise ValueError(f'The file format {extension} is not supported!')
+                    logger.error(f"The file format {extension} is not supported!")
+                    raise ValueError(f"The file format {extension} is not supported!")
 
                 table = FileTable(self, table_name=table_name)
                 df = table.select(query)
 
-            response = Response(
-                RESPONSE_TYPE.TABLE,
-                data_frame=df
-            )
+            response = Response(RESPONSE_TYPE.TABLE, data_frame=df)
         elif isinstance(query, Insert):
             table_name = query.table.parts[-1]
             table = FileTable(self, table_name=table_name)
@@ -364,7 +361,7 @@ class S3Handler(APIHandler):
             scan_buckets = [self.bucket]
         else:
             add_bucket_to_name = True
-            scan_buckets = [b['Name'] for b in client.list_buckets()['Buckets']]
+            scan_buckets = [b["Name"] for b in client.list_buckets()["Buckets"]]
 
         objects = []
         for bucket in scan_buckets:
@@ -372,23 +369,38 @@ class S3Handler(APIHandler):
                 continue
 
             resp = client.list_objects_v2(Bucket=bucket)
-            if 'Contents' not in resp:
+            if "Contents" not in resp:
                 continue
 
-            for obj in resp['Contents']:
-                if obj.get('StorageClass', 'STANDARD') != 'STANDARD':
+            for obj in resp["Contents"]:
+                if obj.get("StorageClass", "STANDARD") != "STANDARD":
                     continue
 
-                obj['Bucket'] = bucket
+                obj["Bucket"] = bucket
                 if add_bucket_to_name:
                     # bucket is part of the name
-                    obj['Key'] = f'{bucket}/{obj["Key"]}'
+                    obj["Key"] = f"{bucket}/{obj['Key']}"
                 objects.append(obj)
             if limit is not None and len(objects) >= limit:
                 break
 
         return objects
 
+    def generate_sas_url(self, key: str, bucket: str) -> str:
+        """
+        Generates a pre-signed URL for accessing an object in the S3 bucket.
+
+        Args:
+            key (str): The key (path) of the object in the S3 bucket.
+            bucket (str): The name of the S3 bucket.
+
+        Returns:
+            str: The pre-signed URL for accessing the object.
+        """
+        client = self.connect()
+        url = client.generate_presigned_url("get_object", Params={"Bucket": bucket, "Key": key}, ExpiresIn=3600)
+        return url
+
     def get_tables(self) -> Response:
         """
         Retrieves a list of tables (objects) in the S3 bucket.
@@ -402,21 +414,13 @@ class S3Handler(APIHandler):
         # Get only the supported file formats.
         # Wrap the object names with backticks to prevent SQL syntax errors.
         supported_names = [
-            f"`{obj['Key']}`"
-            for obj in self.get_objects()
-            if obj['Key'].split('.')[-1] in self.supported_file_formats
+            f"`{obj['Key']}`" for obj in self.get_objects() if obj["Key"].split(".")[-1] in self.supported_file_formats
         ]
 
         # virtual table with list of files
-        supported_names.insert(0, 'files')
+        supported_names.insert(0, "files")
 
-        response = Response(
-            RESPONSE_TYPE.TABLE,
-            data_frame=pd.DataFrame(
-                supported_names,
-                columns=['table_name']
-            )
-        )
+        response = Response(RESPONSE_TYPE.TABLE, data_frame=pd.DataFrame(supported_names, columns=["table_name"]))
 
         return response
 
@@ -433,11 +437,7 @@ class S3Handler(APIHandler):
         Returns:
             Response: A response object containing the column details, formatted as per the `Response` class.
         """
-        query = Select(
-            targets=[Star()],
-            from_table=Identifier(parts=[table_name]),
-            limit=Constant(1)
-        )
+        query = Select(targets=[Star()], from_table=Identifier(parts=[table_name]), limit=Constant(1))
 
         result = self.query(query)
 
@@ -445,10 +445,12 @@ class S3Handler(APIHandler):
             RESPONSE_TYPE.TABLE,
             data_frame=pd.DataFrame(
                 {
-                    'column_name': result.data_frame.columns,
-                    'data_type': [data_type if data_type != 'object' else 'string' for data_type in result.data_frame.dtypes]
+                    "column_name": result.data_frame.columns,
+                    "data_type": [
+                        data_type if data_type != "object" else "string" for data_type in result.data_frame.dtypes
+                    ],
                 }
-            )
+            ),
         )
 
         return response

mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py

@@ -72,7 +72,7 @@ class SalesforceHandler(MetaAPIHandler):
 
             resource_tables = self._get_resource_names()
             for resource_name in resource_tables:
-                table_class = create_table_class(resource_name)
+                table_class = create_table_class(resource_name.lower())
                 self._register_table(resource_name, table_class(self))
 
             return self.connection
@@ -271,10 +271,11 @@ class SalesforceHandler(MetaAPIHandler):
 
         # Retrieve the metadata for all Salesforce resources.
         main_metadata = connection.sobjects.describe()
-
         if table_names:
             # Filter the metadata for the specified tables.
-            main_metadata = [resource for resource in main_metadata["sobjects"] if resource["name"] in table_names]
+            main_metadata = [
+                resource for resource in main_metadata["sobjects"] if resource["name"].lower() in table_names
+            ]
         else:
             main_metadata = main_metadata["sobjects"]
 

mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py

@@ -164,9 +164,18 @@ def create_table_class(resource_name: Text) -> MetaAPIResource:
             """
             client = self.handler.connect()
 
-            resource_metadata = next(
-                (resource for resource in main_metadata if resource["name"] == resource_name),
-            )
+            try:
+                resource_metadata = next(
+                    (resource for resource in main_metadata if resource["name"].lower() == resource_name),
+                )
+            except Exception as e:
+                logger.warning(f"Failed to get resource metadata for {resource_name}: {e}")
+                return {
+                    "table_name": table_name,
+                    "table_type": "BASE TABLE",
+                    "table_description": "",
+                    "row_count": None,
+                }
 
             # Get row count if Id column is aggregatable.
             row_count = None