MindsDB 25.7.3.0__py3-none-any.whl → 25.7.4.0__py3-none-any.whl

mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py

@@ -11,6 +11,7 @@ from mindsdb.integrations.libs.response import (
     RESPONSE_TYPE,
 )
 from mindsdb.integrations.handlers.salesforce_handler.salesforce_tables import create_table_class
+from mindsdb.integrations.handlers.salesforce_handler.constants import get_soql_instructions
 from mindsdb.utilities import log
 
 
@@ -156,91 +157,152 @@ class SalesforceHandler(MetaAPIHandler):
 
     def _get_resource_names(self) -> List[str]:
         """
-        Retrieves the names of the Salesforce resources, with more aggressive filtering to remove tables.
+        Retrieves the names of the Salesforce resources with optimized pre-filtering.
         Returns:
             List[str]: A list of filtered resource names.
         """
         if not self.resource_names:
-            all_resources = [
-                resource["name"]
-                for resource in self.connection.sobjects.describe()["sobjects"]
-                if resource.get("queryable", False)
-            ]
+            # Check for user-specified table filtering first
+            include_tables = self.connection_data.get("include_tables") or self.connection_data.get("tables")
+            exclude_tables = self.connection_data.get("exclude_tables", [])
+
+            if include_tables:
+                # OPTIMIZATION: Skip expensive global describe() call
+                # Only validate the specified tables
+                logger.info(f"Using pre-filtered table list: {include_tables}")
+                self.resource_names = self._validate_specified_tables(include_tables, exclude_tables)
+            else:
+                # Fallback to full discovery with hard-coded filtering
+                logger.info("No table filter specified, performing full discovery...")
+                self.resource_names = self._discover_all_tables_with_filtering(exclude_tables)
 
-            # Define patterns for tables to be filtered out.
-            # Expanded suffixes and prefixes and exact matches
-            ignore_suffixes = ("Share", "History", "Feed", "ChangeEvent", "Tag", "Permission", "Setup", "Consent")
-            ignore_prefixes = (
-                "Apex",
-                "CommPlatform",
-                "Lightning",
-                "Flow",
-                "Transaction",
-                "AI",
-                "Aura",
-                "ContentWorkspace",
-                "Collaboration",
-                "Datacloud",
-            )
-            ignore_exact = {
-                "EntityDefinition",
-                "FieldDefinition",
-                "RecordType",
-                "CaseStatus",
-                "UserRole",
-                "UserLicense",
-                "UserPermissionAccess",
-                "UserRecordAccess",
-                "Folder",
-                "Group",
-                "Note",
-                "ProcessDefinition",
-                "ProcessInstance",
-                "ContentFolder",
-                "ContentDocumentSubscription",
-                "DashboardComponent",
-                "Report",
-                "Dashboard",
-                "Topic",
-                "TopicAssignment",
-                "Period",
-                "Partner",
-                "PackageLicense",
-                "ColorDefinition",
-                "DataUsePurpose",
-                "DataUseLegalBasis",
-            }
-
-            ignore_substrings = (
-                "CleanInfo",
-                "Template",
-                "Rule",
-                "Definition",
-                "Status",
-                "Policy",
-                "Setting",
-                "Access",
-                "Config",
-                "Subscription",
-                "DataType",
-                "MilestoneType",
-                "Entitlement",
-                "Auth",
-            )
-
-            filtered = []
-            for r in all_resources:
-                if (
-                    not r.endswith(ignore_suffixes)
-                    and not r.startswith(ignore_prefixes)
-                    and not any(sub in r for sub in ignore_substrings)
-                    and r not in ignore_exact
-                ):
-                    filtered.append(r)
-
-            self.resource_names = [r for r in filtered]
         return self.resource_names
 
+    def _validate_specified_tables(self, include_tables: List[str], exclude_tables: List[str]) -> List[str]:
+        """
+        Validate user-specified tables without expensive global describe() call.
+
+        Args:
+            include_tables: List of table names to include
+            exclude_tables: List of table names to exclude
+
+        Returns:
+            List[str]: Validated and filtered table names
+        """
+        validated_tables = []
+
+        for table_name in include_tables:
+            # Skip if explicitly excluded
+            if table_name in exclude_tables:
+                logger.info(f"Skipping excluded table: {table_name}")
+                continue
+
+            try:
+                # Quick validation: check if table exists and is queryable
+                # This is much faster than global describe()
+                metadata = getattr(self.connection.sobjects, table_name).describe()
+                if metadata.get("queryable", False):
+                    validated_tables.append(table_name)
+                    logger.debug(f"Validated table: {table_name}")
+                else:
+                    logger.warning(f"Table {table_name} is not queryable, skipping")
+            except Exception as e:
+                logger.warning(f"Table {table_name} not found or accessible: {e}")
+
+        logger.info(f"Validated {len(validated_tables)} tables from include_tables")
+        return validated_tables
+
+    def _discover_all_tables_with_filtering(self, exclude_tables: List[str]) -> List[str]:
+        """
+        Fallback method: discover all tables with hard-coded filtering.
+
+        Args:
+            exclude_tables: List of table names to exclude
+
+        Returns:
+            List[str]: Filtered table names
+        """
+        # This is the original expensive approach - only used when no include_tables specified
+        all_resources = [
+            resource["name"]
+            for resource in self.connection.sobjects.describe()["sobjects"]
+            if resource.get("queryable", False)
+        ]
+
+        # Apply hard-coded filtering (existing logic)
+        ignore_suffixes = ("Share", "History", "Feed", "ChangeEvent", "Tag", "Permission", "Setup", "Consent")
+        ignore_prefixes = (
+            "Apex",
+            "CommPlatform",
+            "Lightning",
+            "Flow",
+            "Transaction",
+            "AI",
+            "Aura",
+            "ContentWorkspace",
+            "Collaboration",
+            "Datacloud",
+        )
+        ignore_exact = {
+            "EntityDefinition",
+            "FieldDefinition",
+            "RecordType",
+            "CaseStatus",
+            "UserRole",
+            "UserLicense",
+            "UserPermissionAccess",
+            "UserRecordAccess",
+            "Folder",
+            "Group",
+            "Note",
+            "ProcessDefinition",
+            "ProcessInstance",
+            "ContentFolder",
+            "ContentDocumentSubscription",
+            "DashboardComponent",
+            "Report",
+            "Dashboard",
+            "Topic",
+            "TopicAssignment",
+            "Period",
+            "Partner",
+            "PackageLicense",
+            "ColorDefinition",
+            "DataUsePurpose",
+            "DataUseLegalBasis",
+        }
+
+        ignore_substrings = (
+            "CleanInfo",
+            "Template",
+            "Rule",
+            "Definition",
+            "Status",
+            "Policy",
+            "Setting",
+            "Access",
+            "Config",
+            "Subscription",
+            "DataType",
+            "MilestoneType",
+            "Entitlement",
+            "Auth",
+        )
+
+        # Apply hard-coded filtering
+        filtered = []
+        for r in all_resources:
+            if (
+                not r.endswith(ignore_suffixes)
+                and not r.startswith(ignore_prefixes)
+                and not any(sub in r for sub in ignore_substrings)
+                and r not in ignore_exact
+                and r not in exclude_tables  # Apply user exclusions
+            ):
+                filtered.append(r)
+
+        return filtered
+
     def meta_get_handler_info(self, **kwargs) -> str:
         """
         Retrieves information about the design and implementation of the API handler.
@@ -254,8 +316,7 @@ class SalesforceHandler(MetaAPIHandler):
         Returns:
             str: A string containing information about the API handler's design and implementation.
         """
-        # TODO: Relationships? Aliases?
-        return "When filtering on a Date or DateTime field, the value MUST be an unquoted literal in YYYY-MM-DD or YYYY-MM-DDThh:mm:ssZ format. For example, CloseDate >= 2025-05-28 is correct; CloseDate >= '2025-05-28' is incorrect."
+        return get_soql_instructions(self.name)
 
     def meta_get_tables(self, table_names: Optional[List[str]] = None) -> Response:
         """

mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py

@@ -176,7 +176,6 @@ def create_table_class(resource_name: Text) -> MetaAPIResource:
                     "table_description": "",
                     "row_count": None,
                 }
-
             # Get row count if Id column is aggregatable.
             row_count = None
             # if next(field for field in resource_metadata['fields'] if field['name'] == 'Id').get('aggregatable', False):

mindsdb/integrations/handlers/tpot_handler/requirements.txt

@@ -1,2 +1,2 @@
 tpot<=0.11.7
-type_infer==0.0.20
+type_infer==0.0.23

mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py

@@ -100,26 +100,25 @@ def parallel_get_all_website_links(urls) -> dict:
         return url_contents
 
     with concurrent.futures.ProcessPoolExecutor() as executor:
-        future_to_url = {
-            executor.submit(get_all_website_links, url): url for url in urls
-        }
+        future_to_url = {executor.submit(get_all_website_links, url): url for url in urls}
         for future in concurrent.futures.as_completed(future_to_url):
             url = future_to_url[future]
             try:
                 url_contents[url] = future.result()
             except Exception as exc:
-                logger.error(f'{url} generated an exception: {exc}')
+                logger.error(f"{url} generated an exception: {exc}")
                 # don't raise the exception, just log it, continue processing other urls
 
     return url_contents
 
 
-def get_all_website_links(url) -> dict:
+def get_all_website_links(url, headers: dict = None) -> dict:
     """
     Fetch all website links from a URL.
 
     Args:
         url (str): the URL to fetch links from
+        headers (dict): a dictionary of headers to use when fetching links
 
     Returns:
         A dictionary containing the URL, the extracted links, the HTML content, the text content, and any error that occurred.
@@ -132,9 +131,12 @@ def get_all_website_links(url) -> dict:
         session = requests.Session()
 
         # Add headers to mimic a real browser request
-        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
-        }
+        if headers is None:
+            headers = {}
+        if "User-Agent" not in headers:
+            headers["User-Agent"] = (
+                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3"
+            )
 
         response = session.get(url, headers=headers)
         if "cookie" in response.request.headers:
@@ -157,7 +159,7 @@ def get_all_website_links(url) -> dict:
                     continue
                 href = urljoin(url, href)
                 parsed_href = urlparse(href)
-                href = urlunparse((parsed_href.scheme, parsed_href.netloc, parsed_href.path, '', '', ''))
+                href = urlunparse((parsed_href.scheme, parsed_href.netloc, parsed_href.path, "", "", ""))
                 if not is_valid(href):
                     continue
                 if href in urls:
@@ -203,7 +205,15 @@ def get_readable_text_from_soup(soup) -> str:
     return html_converter.handle(str(soup))
 
 
-def get_all_website_links_recursively(url, reviewed_urls, limit=None, crawl_depth: int = 1, current_depth: int = 0, filters: List[str] = None):
+def get_all_website_links_recursively(
+    url,
+    reviewed_urls,
+    limit=None,
+    crawl_depth: int = 1,
+    current_depth: int = 0,
+    filters: List[str] = None,
+    headers=None,
+):
     """
     Recursively gathers all links from a given website up to a specified limit.
 
@@ -227,7 +237,7 @@ def get_all_website_links_recursively(url, reviewed_urls, limit=None, crawl_dept
         matches_filter = any(re.match(f, url) is not None for f in filters)
     if url not in reviewed_urls and matches_filter:
         try:
-            reviewed_urls[url] = get_all_website_links(url)
+            reviewed_urls[url] = get_all_website_links(url, headers=headers)
         except Exception as e:
             error_message = traceback.format_exc().splitlines()[-1]
             logger.error("An exception occurred: %s", str(e))
@@ -271,10 +281,14 @@ def get_all_website_links_recursively(url, reviewed_urls, limit=None, crawl_dept
         reviewed_urls.update(new_revised_urls)
 
         for new_url in new_revised_urls:
-            get_all_website_links_recursively(new_url, reviewed_urls, limit, crawl_depth=crawl_depth, current_depth=current_depth + 1, filters=filters)
+            get_all_website_links_recursively(
+                new_url, reviewed_urls, limit, crawl_depth=crawl_depth, current_depth=current_depth + 1, filters=filters
+            )
 
 
-def get_all_websites(urls, limit=1, html=False, crawl_depth: int = 1, filters: List[str] = None) -> pd.DataFrame:
+def get_all_websites(
+    urls, limit=1, html=False, crawl_depth: int = 1, filters: List[str] = None, headers: dict = None
+) -> pd.DataFrame:
     """
     Crawl a list of websites and return a DataFrame containing the results.
 
@@ -284,6 +298,7 @@ def get_all_websites(urls, limit=1, html=False, crawl_depth: int = 1, filters: L
         crawl_depth (int): Crawl depth for URLs.
         html (bool): a boolean indicating whether to include the HTML content in the results
         filters (List[str]): Crawl URLs that only match these regex patterns.
+        headers (dict): headers of request
 
     Returns:
         A DataFrame containing the results.
@@ -299,7 +314,9 @@ def get_all_websites(urls, limit=1, html=False, crawl_depth: int = 1, filters: L
         if urlparse(url).scheme == "":
             # Try HTTPS first
             url = "https://" + url
-        get_all_website_links_recursively(url, reviewed_urls, limit, crawl_depth=crawl_depth, filters=filters)
+        get_all_website_links_recursively(
+            url, reviewed_urls, limit, crawl_depth=crawl_depth, filters=filters, headers=headers
+        )
 
     # Use a ThreadPoolExecutor to run the helper function in parallel.
     with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -311,9 +328,7 @@ def get_all_websites(urls, limit=1, html=False, crawl_depth: int = 1, filters: L
     columns_to_ignore = ["urls"]
     if html is False:
         columns_to_ignore += ["html_content"]
-    df = dict_to_dataframe(
-        reviewed_urls, columns_to_ignore=columns_to_ignore, index_name="url"
-    )
+    df = dict_to_dataframe(reviewed_urls, columns_to_ignore=columns_to_ignore, index_name="url")
 
     if not df.empty and df[df.error.isna()].empty:
         raise Exception(str(df.iloc[0].error))

mindsdb/integrations/handlers/web_handler/web_handler.py

@@ -7,17 +7,11 @@ from mindsdb.utilities.security import validate_urls
 from .urlcrawl_helpers import get_all_websites
 
 from mindsdb.integrations.libs.api_handler import APIResource, APIHandler
-from mindsdb.integrations.utilities.sql_utils import (FilterCondition, FilterOperator)
+from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
 
 
 class CrawlerTable(APIResource):
-
-    def list(
-            self,
-            conditions: List[FilterCondition] = None,
-            limit: int = None,
-            **kwargs
-    ) -> pd.DataFrame:
+    def list(self, conditions: List[FilterCondition] = None, limit: int = None, **kwargs) -> pd.DataFrame:
         """
         Selects data from the provided websites
 
@@ -30,27 +24,34 @@ class CrawlerTable(APIResource):
         urls = []
         crawl_depth = None
         per_url_limit = None
+        headers = {}
         for condition in conditions:
-            if condition.column == 'url':
+            if condition.column == "url":
                 if condition.op == FilterOperator.IN:
                     urls = condition.value
                 elif condition.op == FilterOperator.EQUAL:
                     urls = [condition.value]
                 condition.applied = True
-            if condition.column == 'crawl_depth' and condition.op == FilterOperator.EQUAL:
+            if condition.column == "crawl_depth" and condition.op == FilterOperator.EQUAL:
                 crawl_depth = condition.value
                 condition.applied = True
-            if condition.column == 'per_url_limit' and condition.op == FilterOperator.EQUAL:
+            if condition.column == "per_url_limit" and condition.op == FilterOperator.EQUAL:
                 per_url_limit = condition.value
                 condition.applied = True
+            if condition.column.lower() == "user_agent" and condition.op == FilterOperator.EQUAL:
+                headers["User-Agent"] = condition.value
+                condition.applied = True
 
         if len(urls) == 0:
             raise NotImplementedError(
-                'You must specify what url you want to crawl, for example: SELECT * FROM web.crawler WHERE url = "someurl"')
+                'You must specify what url you want to crawl, for example: SELECT * FROM web.crawler WHERE url = "someurl"'
+            )
 
-        allowed_urls = config.get('web_crawling_allowed_sites', [])
+        allowed_urls = config.get("web_crawling_allowed_sites", [])
         if allowed_urls and not validate_urls(urls, allowed_urls):
-            raise ValueError(f"The provided URL is not allowed for web crawling. Please use any of {', '.join(allowed_urls)}.")
+            raise ValueError(
+                f"The provided URL is not allowed for web crawling. Please use any of {', '.join(allowed_urls)}."
+            )
 
         if limit is None and per_url_limit is None and crawl_depth is None:
             per_url_limit = 1
@@ -58,10 +59,10 @@ class CrawlerTable(APIResource):
             # crawl every url separately
             results = []
             for url in urls:
-                results.append(get_all_websites([url], per_url_limit, crawl_depth=crawl_depth))
+                results.append(get_all_websites([url], per_url_limit, crawl_depth=crawl_depth, headers=headers))
             result = pd.concat(results)
         else:
-            result = get_all_websites(urls, limit, crawl_depth=crawl_depth)
+            result = get_all_websites(urls, limit, crawl_depth=crawl_depth, headers=headers)
 
         if limit is not None and len(result) > limit:
             result = result[:limit]
@@ -72,11 +73,7 @@ class CrawlerTable(APIResource):
         """
         Returns the columns of the crawler table
         """
-        return [
-            'url',
-            'text_content',
-            'error'
-        ]
+        return ["url", "text_content", "error"]
 
 
 class WebHandler(APIHandler):
@@ -87,7 +84,7 @@ class WebHandler(APIHandler):
     def __init__(self, name=None, **kwargs):
         super().__init__(name)
         crawler = CrawlerTable(self)
-        self._register_table('crawler', crawler)
+        self._register_table("crawler", crawler)
 
     def check_connection(self) -> HandlerStatusResponse:
         """

mindsdb/integrations/libs/vectordatabase_handler.py

@@ -334,12 +334,21 @@ class VectorStoreHandler(BaseHandler):
 
         if not df_update.empty:
             # get values of existed `created_at` and return them to metadata
-            created_dates = {row[id_col]: row[metadata_col].get("_created_at") for _, row in df_existed.iterrows()}
+            origin_id_col = "_original_doc_id"
+
+            created_dates, ids = {}, {}
+            for _, row in df_existed.iterrows():
+                chunk_id = row[id_col]
+                created_dates[chunk_id] = row[metadata_col].get("_created_at")
+                ids[chunk_id] = row[metadata_col].get(origin_id_col)
 
             def keep_created_at(row):
                 val = created_dates.get(row[id_col])
                 if val:
                     row[metadata_col]["_created_at"] = val
+                # keep id column
+                if origin_id_col not in row[metadata_col]:
+                    row[metadata_col][origin_id_col] = ids.get(row[id_col])
                 return row
 
             df_update.apply(keep_created_at, axis=1)

mindsdb/integrations/utilities/handler_utils.py

@@ -37,54 +37,74 @@ def get_api_key(
 
     # 1
     if "using" in create_args and f"{api_name.lower()}_api_key" in create_args["using"]:
-        return create_args["using"][f"{api_name.lower()}_api_key"]
+        api_key = create_args["using"][f"{api_name.lower()}_api_key"]
+        if api_key:
+            return api_key
 
     # 1.5 - Check for generic api_key in using
     if "using" in create_args and "api_key" in create_args["using"]:
-        return create_args["using"]["api_key"]
+        api_key = create_args["using"]["api_key"]
+        if api_key:
+            return api_key
 
     # 2
     if f"{api_name.lower()}_api_key" in create_args:
-        return create_args[f"{api_name.lower()}_api_key"]
+        api_key = create_args[f"{api_name.lower()}_api_key"]
+        if api_key:
+            return api_key
 
     # 2.5 - Check for generic api_key
     if "api_key" in create_args:
-        return create_args["api_key"]
+        api_key = create_args["api_key"]
+        if api_key:
+            return api_key
 
     # 3 - Check in params dictionary if it exists (for agents)
     if "params" in create_args and create_args["params"] is not None:
         if f"{api_name.lower()}_api_key" in create_args["params"]:
-            return create_args["params"][f"{api_name.lower()}_api_key"]
+            api_key = create_args["params"][f"{api_name.lower()}_api_key"]
+            if api_key:
+                return api_key
         # 3.5 - Check for generic api_key in params
         if "api_key" in create_args["params"]:
-            return create_args["params"]["api_key"]
+            api_key = create_args["params"]["api_key"]
+            if api_key:
+                return api_key
 
     # 4
     if engine_storage is not None:
         connection_args = engine_storage.get_connection_args()
         if f"{api_name.lower()}_api_key" in connection_args:
-            return connection_args[f"{api_name.lower()}_api_key"]
+            api_key = connection_args[f"{api_name.lower()}_api_key"]
+            if api_key:
+                return api_key
         # 4.5 - Check for generic api_key in connection_args
         if "api_key" in connection_args:
-            return connection_args["api_key"]
+            api_key = connection_args["api_key"]
+            if api_key:
+                return api_key
 
     # 5
     api_key = os.getenv(f"{api_name.lower()}_api_key")
-    if api_key is not None:
+    if api_key:
         return api_key
     api_key = os.getenv(f"{api_name.upper()}_API_KEY")
-    if api_key is not None:
+    if api_key:
         return api_key
 
     # 6
     config = Config()
     api_cfg = config.get(api_name, {})
     if f"{api_name.lower()}_api_key" in api_cfg:
-        return api_cfg[f"{api_name.lower()}_api_key"]
+        api_key = api_cfg[f"{api_name.lower()}_api_key"]
+        if api_key:
+            return api_key
 
     # 7
     if "api_keys" in create_args and api_name in create_args["api_keys"]:
-        return create_args["api_keys"][api_name]
+        api_key = create_args["api_keys"][api_name]
+        if api_key:
+            return api_key
 
     if strict:
         provider_upper = api_name.upper()