MindsDB 25.3.2.0__py3-none-any.whl → 25.3.3.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.3.2.0'
+__version__ = '25.3.3.0'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/__main__.py

@@ -46,7 +46,6 @@ try:
 except RuntimeError:
     logger.info('Torch multiprocessing context already set, ignoring...')
 
-
 _stop_event = threading.Event()
 
 

mindsdb/integrations/handlers/github_handler/generate_api.py

@@ -0,0 +1,228 @@
+import inspect
+from typing import List
+from dataclasses import dataclass
+
+import pandas as pd
+import github
+
+from mindsdb.integrations.utilities.sql_utils import (FilterCondition, FilterOperator, SortColumn)
+from mindsdb.integrations.libs.api_handler import APIResource
+
+
+@dataclass
+class Type:
+    name: str
+    sub_type: str = None
+    optional: bool = False
+
+
+@dataclass
+class GHMethod:
+    name: str
+    table_name: str
+    params: dict
+    output: Type
+
+
+def parse_annotations(annotations):
+    '''
+    Parse string annotation, and extract type, input examples:
+    - Milestone | Opt[str]
+    - PaginatedList[Issue]
+    '''
+    type_name, sub_type = None, None
+    if not isinstance(annotations, str):
+
+        return Type(getattr(annotations, '__name__', None))
+    for item in annotations.split('|'):
+        item = item.strip()
+        if item is None:
+            continue
+        if '[' in item:
+            type_name = item[: item.find('[')]
+            item2 = item[item.find('[') + 1: item.rfind(']')]
+            if type_name == 'Opt':
+                inner_type = parse_annotations(item2)
+                inner_type.optional = Type
+                return inner_type
+            if type_name == 'dict':
+                item2 = item2[item2.find(',') + 1:]
+            sub_type = parse_annotations(item2).name
+        else:
+            type_name = item
+        # get only first type
+        break
+    return Type(type_name, sub_type)
+
+
+def get_properties(cls):
+    # find properties of the class
+
+    properties = {}
+    for prop_name, prop in inspect.getmembers(cls):
+        if prop_name.startswith('_'):
+            continue
+        if not isinstance(prop, property):
+            continue
+        sig2 = inspect.signature(prop.fget)
+
+        properties[prop_name] = parse_annotations(sig2.return_annotation)
+    return properties
+
+
+def get_github_types():
+    # get github types
+    types = {}
+
+    GithubObject = github.GithubObject.GithubObject
+    for module_name, module in inspect.getmembers(github, inspect.ismodule):
+        cls = getattr(module, module_name, None)
+        if cls is None:
+            continue
+        if issubclass(cls, GithubObject):
+
+            # remove inherited props
+            parent_props = []
+            for cls2 in cls.__bases__:
+                parent_props += get_properties(cls2).keys()
+
+            properties = {}
+            for k, v in get_properties(cls).items():
+                if k not in parent_props:
+                    properties[k] = v
+
+            types[module_name] = properties
+    return types
+
+
+def get_github_methods(cls):
+    '''
+    Analyse class in order to find methods which return list of objects.
+    '''
+    methods = []
+
+    for method_name, method in inspect.getmembers(cls, inspect.isfunction):
+        sig = inspect.signature(method)
+
+        return_type = parse_annotations(sig.return_annotation)
+        list_prefix = 'get_'
+        if not (method_name.startswith(list_prefix) and return_type.name == 'PaginatedList'):
+            continue
+
+        table_name = method_name[len(list_prefix):]
+
+        params = {}
+        for param_name, param in sig.parameters.items():
+            params[param_name] = parse_annotations(param.annotation)
+
+        methods.append(GHMethod(
+            name=method_name,
+            table_name=table_name,
+            params=params,
+            output=return_type
+        ))
+    return methods
+
+
+class GHTable(APIResource):
+    def __init__(self, *args, method: GHMethod = None, github_types=None, **kwargs):
+        self.method = method
+        self.github_types = github_types
+
+        self.output_columns = {}
+        if method.output.sub_type in self.github_types:
+            self.output_columns = self.github_types[method.output.sub_type]
+
+        # check params:
+        self.params, self.list_params = [], []
+        for name, param_type in method.params.items():
+            self.params.append(name)
+            if param_type.name == 'list':
+                self.list_params.append(name)
+
+        self._allow_sort = 'sort' in method.params
+
+        super().__init__(*args, **kwargs)
+
+    def repr_value(self, value, type_name):
+        if value is None or type_name in ('bool', 'int', 'float'):
+            return value
+        if type_name in self.github_types:
+            properties = self.github_types[type_name]
+            if 'login' in properties:
+                value = getattr(value, 'login')
+            elif 'url' in properties:
+                value = getattr(value, 'url')
+        return str(value)
+
+    def get_columns(self) -> List[str]:
+        return list(self.output_columns.keys())
+
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ) -> pd.DataFrame:
+
+        if limit is None:
+            limit = 20
+
+        method_kwargs = {}
+        if sort is not None and self._allow_sort:
+            for col in sort:
+                method_kwargs['sort'] = col.column
+                method_kwargs['direction'] = 'asc' if col.ascending else 'desc'
+                sort.applied = True
+                # supported only 1 column
+                break
+
+        if conditions:
+            for condition in conditions:
+                if condition.column not in self.params:
+                    continue
+
+                if condition.column in self.list_params:
+                    if condition.op == FilterOperator.IN:
+                        method_kwargs[condition.column] = condition.value
+                    elif condition.op == FilterOperator.EQUAL:
+                        method_kwargs[condition.column] = [condition]
+                    condition.applied = True
+                else:
+                    method_kwargs[condition.column] = condition.value
+                    condition.applied = True
+
+        connection = self.handler.connect()
+        method = getattr(connection.get_repo(self.handler.repository), self.method.name)
+
+        data = []
+        count = 0
+        for record in method(**method_kwargs):
+            item = {}
+            for name, output_type in self.output_columns.items():
+
+                # workaround to prevent making addition request per property.
+                if name in targets:
+                    # request only if is required
+                    value = getattr(record, name)
+                else:
+                    value = getattr(record, '_' + name).value
+                if value is not None:
+                    if output_type.name == 'list':
+                        value = ",".join([
+                                str(self.repr_value(i, output_type.sub_type))
+                                for i in value
+                        ])
+                    else:
+                        value = self.repr_value(value, output_type.name)
+                item[name] = value
+
+            data.append(item)
+
+            count += 1
+            if limit <= count:
+                break
+
+        return pd.DataFrame(data, columns=self.get_columns())

mindsdb/integrations/handlers/github_handler/github_handler.py

@@ -12,7 +12,7 @@ from mindsdb.integrations.handlers.github_handler.github_tables import (
     GithubMilestonesTable,
     GithubProjectsTable, GithubFilesTable
 )
-
+from mindsdb.integrations.handlers.github_handler.generate_api import get_github_types, get_github_methods, GHTable
 from mindsdb.integrations.libs.api_handler import APIHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
@@ -22,6 +22,7 @@ from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
 
+
 class GithubHandler(APIHandler):
     """The GitHub handler implementation"""
 
@@ -43,16 +44,22 @@ class GithubHandler(APIHandler):
         self.connection = None
         self.is_connected = False
 
+        # custom tables
         self._register_table("issues", GithubIssuesTable(self))
-        self._register_table("pull_requests", GithubPullRequestsTable(self))
-        self._register_table("commits", GithubCommitsTable(self))
-        self._register_table("releases", GithubReleasesTable(self))
-        self._register_table("branches", GithubBranchesTable(self))
-        self._register_table("contributors", GithubContributorsTable(self))
-        self._register_table("milestones", GithubMilestonesTable(self))
-        self._register_table("projects", GithubProjectsTable(self))
         self._register_table("files", GithubFilesTable(self))
 
+        # generated tables
+        github_types = get_github_types()
+
+        # generate tables from repository object
+        for method in get_github_methods(github.Repository.Repository):
+            if method.table_name in self._tables:
+                continue
+
+            table = GHTable(self, github_types=github_types, method=method)
+            self._register_table(method.table_name, table)
+
+
     def connect(self) -> StatusResponse:
         """Set up the connection required by the handler.
 

mindsdb/integrations/handlers/github_handler/requirements.txt

@@ -1 +1 @@
-pygithub
+pygithub==2.6.1

mindsdb/integrations/handlers/redshift_handler/redshift_handler.py

@@ -57,6 +57,7 @@ class RedshiftHandler(PostgresHandler):
                 connection.commit()
             except Exception as e:
                 logger.error(f"Error inserting data into {table_name}, {e}!")
+                connection.rollback()
                 response = Response(
                     RESPONSE_TYPE.ERROR,
                     error_code=0,

mindsdb/integrations/handlers/salesforce_handler/requirements.txt

@@ -1 +1 @@
-salesforce_api
+salesforce_api==0.1.45

mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py

@@ -39,31 +39,8 @@ class SalesforceHandler(APIHandler):
 
         self.connection = None
         self.is_connected = False
-
-        # Register Salesforce tables.
-        self.resource_names = {
-            'Account',
-            'Contact',
-            'Opportunity',
-            'Lead',
-            'Task',
-            'Event',
-            'User',
-            'Product2',
-            'Pricebook2',
-            'PricebookEntry',
-            'Order',
-            'OrderItem',
-            'Case',
-            'Campaign',
-            'CampaignMember',
-            'Contract',
-            'Asset'
-        }
-
-        for resource_name in self.resource_names:
-            table_class = create_table_class(resource_name, resource_name)
-            self._register_table(resource_name, table_class(self))
+        self.thread_safe = True
+        self.resource_names = []
 
     def connect(self) -> salesforce_api.client.Client:
         """
@@ -92,6 +69,12 @@ class SalesforceHandler(APIHandler):
                 is_sandbox=self.connection_data.get('is_sandbox', False)
             )
             self.is_connected = True
+
+            # Register Salesforce tables.
+            for resource_name in self._get_resource_names():
+                table_class = create_table_class(resource_name)
+                self._register_table(resource_name.lower(), table_class(self))
+
             return self.connection
         except AuthenticationError as auth_error:
             logger.error(f"Authentication error connecting to Salesforce, {auth_error}!")
@@ -179,3 +162,15 @@ class SalesforceHandler(APIHandler):
             )
 
         return response
+
+    def _get_resource_names(self) -> None:
+        """
+        Retrieves the names of the Salesforce resources.
+
+        Returns:
+            None
+        """
+        if not self.resource_names:
+            self.resource_names = [resource['name'] for resource in self.connection.sobjects.describe()['sobjects']]
+
+        return self.resource_names

mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py

@@ -11,7 +11,7 @@ from mindsdb.utilities import log
 logger = log.getLogger(__name__)
 
 
-def create_table_class(table_name: Text, resource_name: Text) -> APIResource:
+def create_table_class(resource_name: Text) -> APIResource:
     """
     Creates a table class for the given Salesforce resource.
     """
@@ -31,7 +31,7 @@ def create_table_class(table_name: Text, resource_name: Text) -> APIResource:
             Returns:
                 pd.DataFrame: A DataFrame containing the data retrieved from the Salesforce resource.
             """
-            query.from_table = table_name
+            query.from_table = resource_name
 
             # SOQL does not support * in SELECT queries. Replace * with column names.
             if isinstance(query.targets[0], Star):

mindsdb/integrations/handlers/timescaledb_handler/timescaledb_handler.py

@@ -10,8 +10,6 @@ class TimeScaleDBHandler(PostgresHandler):
         super().__init__(name, **kwargs)
 
 
-
-
 connection_args = OrderedDict(
     host={
         'type': ARG_TYPE.STR,
@@ -31,6 +29,12 @@ connection_args = OrderedDict(
         'type': ARG_TYPE.STR,
         'description': 'The password to authenticate the user with the TimeScaleDB server.'
     },
+    schema={
+        'type': ARG_TYPE.STR,
+        'description': 'The schema in which objects are searched first.',
+        'required': False,
+        'label': 'Schema'
+    },
     port={
         'type': ARG_TYPE.INT,
         'description': 'Specify port to connect TimeScaleDB '
@@ -39,8 +43,9 @@ connection_args = OrderedDict(
 
 connection_args_example = OrderedDict(
     host='127.0.0.1',
-    port=36806,
-    password='P455W0rD',
-    user='tsdbadmin',
-    database="tsdb"
+    port=5432,
+    password='password',
+    user='root',
+    database="timescaledb",
+    schema='public'
 )

mindsdb/integrations/libs/ml_handler_process/learn_process.py

@@ -111,10 +111,16 @@ def learn_process(data_integration_ref: dict, problem_definition: dict, fetch_da
             )
             handlers_cacher[predictor_record.id] = ml_handler
 
-            if not ml_handler.generative:
+            if not ml_handler.generative and target is not None:
                 if training_data_df is not None and target not in training_data_df.columns:
-                    raise Exception(
-                        f'Prediction target "{target}" not found in training dataframe: {list(training_data_df.columns)}')
+                    # is the case different? convert column case in input dataframe
+                    col_names = {c.lower(): c for c in training_data_df.columns}
+                    target_found = col_names.get(target.lower())
+                    if target_found:
+                        training_data_df.rename(columns={target_found: target}, inplace=True)
+                    else:
+                        raise Exception(
+                            f'Prediction target "{target}" not found in training dataframe: {list(training_data_df.columns)}')
 
             # create new model
             if base_model_id is None:

mindsdb/utilities/render/sqlalchemy_render.py

@@ -483,7 +483,7 @@ class SqlalchemyRender:
 
         return schema, table_name
 
-    def to_table(self, node):
+    def to_table(self, node, is_lateral=False):
         if isinstance(node, ast.Identifier):
             schema, table_name = self.get_table_name(node)
 
@@ -497,7 +497,10 @@ class SqlalchemyRender:
             alias = None
             if node.alias:
                 alias = self.get_alias(node.alias)
-            table = sub_stmt.subquery(alias)
+            if is_lateral:
+                table = sub_stmt.lateral(alias)
+            else:
+                table = sub_stmt.subquery(alias)
 
         else:
             # TODO tests are failing
@@ -526,8 +529,11 @@ class SqlalchemyRender:
 
                 query = query.add_cte(stmt.cte(self.get_alias(alias), nesting=True))
 
-        if node.distinct:
+        if node.distinct is True:
             query = query.distinct()
+        elif isinstance(node.distinct, list):
+            columns = [self.to_expression(c) for c in node.distinct]
+            query = query.distinct(*columns)
 
         if node.from_table is not None:
             from_table = node.from_table
@@ -541,7 +547,8 @@ class SqlalchemyRender:
                 # other tables
                 has_explicit_join = False
                 for item in join_list[1:]:
-                    table = self.to_table(item['table'])
+                    join_type = item['join_type']
+                    table = self.to_table(item['table'], is_lateral=('LATERAL' in join_type))
                     if item['is_implicit']:
                         # add to from clause
                         if has_explicit_join:
@@ -558,7 +565,6 @@ class SqlalchemyRender:
                         else:
                             condition = self.to_expression(item['condition'])
 
-                        join_type = item['join_type']
                         if 'ASOF' in join_type:
                             raise NotImplementedError(f'Unsupported join type: {join_type}')
                         method = 'join'