MindsDB 25.3.1.0__py3-none-any.whl → 25.3.3.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.3.1.0'
+__version__ = '25.3.3.0'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/__main__.py

@@ -9,10 +9,10 @@ import secrets
 import traceback
 import threading
 from enum import Enum
-from packaging import version
 from dataclasses import dataclass, field
 from typing import Callable, Optional, Tuple, List
 
+from packaging import version
 from sqlalchemy.orm.attributes import flag_modified
 
 from mindsdb.utilities import log
@@ -46,7 +46,6 @@ try:
 except RuntimeError:
     logger.info('Torch multiprocessing context already set, ignoring...')
 
-
 _stop_event = threading.Event()
 
 

mindsdb/api/executor/sql_query/steps/union_step.py

@@ -1,9 +1,9 @@
-import hashlib
-
 from mindsdb.api.executor.planner.steps import UnionStep
 
 from mindsdb.api.executor.sql_query.result_set import ResultSet
 from mindsdb.api.executor.exceptions import WrongArgumentError
+from mindsdb.api.executor.utilities.sql import query_df_with_type_infer_fallback
+import numpy as np
 
 from .base import BaseStepCall
 
@@ -30,27 +30,24 @@ class UnionStepCall(BaseStepCall):
         #         if type1 != type2:
         #             raise ErSqlWrongArguments(f'UNION types mismatch: {type1} != {type2}')
 
-        data = ResultSet()
-        for col in left_result.columns:
-            data.add_column(col)
-
-        records_hashes = []
-        results = []
-        for row in left_result.to_lists():
-            if step.unique:
-                checksum = hashlib.sha256(str(row).encode()).hexdigest()
-                if checksum in records_hashes:
-                    continue
-                records_hashes.append(checksum)
-            results.append(list(row))
-
-        for row in right_result.to_lists():
-            if step.unique:
-                checksum = hashlib.sha256(str(row).encode()).hexdigest()
-                if checksum in records_hashes:
-                    continue
-                records_hashes.append(checksum)
-            results.append(list(row))
-        data.add_raw_values(results)
+        table_a, names = left_result.to_df_cols()
+        table_b, _ = right_result.to_df_cols()
+
+        op = 'UNION ALL'
+        if step.unique:
+            op = 'UNION'
+        query = f"""
+            SELECT * FROM table_a
+            {op}
+            SELECT * FROM table_b
+        """
+
+        resp_df, _description = query_df_with_type_infer_fallback(query, {
+            'table_a': table_a,
+            'table_b': table_b
+        })
+        resp_df.replace({np.nan: None}, inplace=True)
+
+        data = ResultSet().from_df_cols(resp_df, col_names=names)
 
         return data

mindsdb/api/http/gui.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from zipfile import ZipFile
 
 import requests
+from packaging.version import Version
 
 from mindsdb.utilities.config import Config
 from mindsdb.utilities import log
@@ -65,7 +66,7 @@ def download_gui(destignation, version):
     """
 
 
-def update_static(gui_version_lv):
+def update_static(gui_version: Version):
     """Update Scout files basing on compatible-config.json content.
     Files will be downloaded and updated if new version of GUI > current.
     Current GUI version stored in static/version.txt.
@@ -74,11 +75,11 @@ def update_static(gui_version_lv):
     static_path = Path(config["paths"]["static"])
 
     logger.info(
-        f"New version of GUI available ({gui_version_lv.vstring}). Downloading..."
+        f"New version of GUI available ({gui_version.base_version}). Downloading..."
     )
 
     temp_dir = tempfile.mkdtemp(prefix="mindsdb_gui_files_")
-    success = download_gui(temp_dir, gui_version_lv.vstring)
+    success = download_gui(temp_dir, gui_version.base_version)
     if success is False:
         shutil.rmtree(temp_dir)
         return False
@@ -90,5 +91,5 @@ def update_static(gui_version_lv):
     shutil.copytree(temp_dir, str(static_path))
     shutil.rmtree(temp_dir_for_rm)
 
-    logger.info(f"GUI version updated to {gui_version_lv.vstring}")
+    logger.info(f"GUI version updated to {gui_version.base_version}")
     return True

mindsdb/api/http/initialize.py

@@ -6,7 +6,6 @@ import traceback
 import webbrowser
 from pathlib import Path
 from http import HTTPStatus
-from distutils.version import LooseVersion
 
 import requests
 from flask import Flask, url_for, make_response, request, send_from_directory
@@ -14,6 +13,7 @@ from flask.json import dumps
 from flask_compress import Compress
 from flask_restx import Api
 from werkzeug.exceptions import HTTPException
+from packaging.version import Version, parse as parse_version
 
 from mindsdb.__about__ import __version__ as mindsdb_version
 from mindsdb.api.http.gui import update_static
@@ -72,7 +72,7 @@ def custom_output_json(data, code, headers=None):
     return resp
 
 
-def get_last_compatible_gui_version() -> LooseVersion:
+def get_last_compatible_gui_version() -> Version:
     logger.debug("Getting last compatible frontend..")
     try:
         res = requests.get('https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json', timeout=5)
@@ -95,7 +95,7 @@ def get_last_compatible_gui_version() -> LooseVersion:
         logger.error(f"Cant decode compatible-config.json: {e}")
         return False
 
-    current_mindsdb_lv = LooseVersion(mindsdb_version)
+    current_mindsdb_lv = parse_version(mindsdb_version)
 
     try:
         gui_versions = {}
@@ -103,31 +103,31 @@ def get_last_compatible_gui_version() -> LooseVersion:
         max_gui_lv = None
         for el in versions['mindsdb']:
             if el['mindsdb_version'] is None:
-                gui_lv = LooseVersion(el['gui_version'])
+                gui_lv = parse_version(el['gui_version'])
             else:
-                mindsdb_lv = LooseVersion(el['mindsdb_version'])
-                gui_lv = LooseVersion(el['gui_version'])
-                if mindsdb_lv.vstring not in gui_versions or gui_lv > gui_versions[mindsdb_lv.vstring]:
-                    gui_versions[mindsdb_lv.vstring] = gui_lv
+                mindsdb_lv = parse_version(el['mindsdb_version'])
+                gui_lv = parse_version(el['gui_version'])
+                if mindsdb_lv.base_version not in gui_versions or gui_lv > gui_versions[mindsdb_lv.base_version]:
+                    gui_versions[mindsdb_lv.base_version] = gui_lv
                 if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
                     max_mindsdb_lv = mindsdb_lv
             if max_gui_lv is None or max_gui_lv < gui_lv:
                 max_gui_lv = gui_lv
 
-        all_mindsdb_lv = [LooseVersion(x) for x in gui_versions.keys()]
+        all_mindsdb_lv = [parse_version(x) for x in gui_versions.keys()]
         all_mindsdb_lv.sort()
 
-        if current_mindsdb_lv.vstring in gui_versions:
-            gui_version_lv = gui_versions[current_mindsdb_lv.vstring]
+        if current_mindsdb_lv.base_version in gui_versions:
+            gui_version_lv = gui_versions[current_mindsdb_lv.base_version]
         elif current_mindsdb_lv > all_mindsdb_lv[-1]:
             gui_version_lv = max_gui_lv
         else:
-            lower_versions = {key: value for key, value in gui_versions.items() if LooseVersion(key) < current_mindsdb_lv}
+            lower_versions = {key: value for key, value in gui_versions.items() if parse_version(key) < current_mindsdb_lv}
             if len(lower_versions) == 0:
-                gui_version_lv = gui_versions[all_mindsdb_lv[0].vstring]
+                gui_version_lv = gui_versions[all_mindsdb_lv[0].base_version]
             else:
-                all_lower_versions = [LooseVersion(x) for x in lower_versions.keys()]
-                gui_version_lv = gui_versions[all_lower_versions[-1].vstring]
+                all_lower_versions = [parse_version(x) for x in lower_versions.keys()]
+                gui_version_lv = gui_versions[all_lower_versions[-1].base_version]
     except Exception as e:
         logger.error(f"Error in compatible-config.json structure: {e}")
         return False
@@ -136,7 +136,7 @@ def get_last_compatible_gui_version() -> LooseVersion:
     return gui_version_lv
 
 
-def get_current_gui_version() -> LooseVersion:
+def get_current_gui_version() -> Version:
     logger.debug("Getting current frontend version..")
     config = Config()
     static_path = Path(config['paths']['static'])
@@ -148,7 +148,7 @@ def get_current_gui_version() -> LooseVersion:
             current_gui_version = f.readline()
 
     current_gui_lv = (
-        None if current_gui_version is None else LooseVersion(current_gui_version)
+        None if current_gui_version is None else parse_version(current_gui_version)
     )
     logger.debug(f"Current frontend version: {current_gui_lv}.")
 
@@ -163,7 +163,7 @@ def initialize_static():
     required_gui_version = config['gui'].get('version')
 
     if required_gui_version is not None:
-        required_gui_version_lv = LooseVersion(required_gui_version)
+        required_gui_version_lv = parse_version(required_gui_version)
         success = True
         if (
             current_gui_version_lv is None
@@ -176,7 +176,7 @@ def initialize_static():
             return False
 
         # ignore versions like '23.9.2.2'
-        if current_gui_version_lv is not None and len(current_gui_version_lv.version) < 3:
+        if current_gui_version_lv is not None and len(current_gui_version_lv.release) < 3:
             if current_gui_version_lv == last_gui_version_lv:
                 return True
         logger.debug("Updating gui..")

mindsdb/integrations/handlers/github_handler/generate_api.py

@@ -0,0 +1,228 @@
+import inspect
+from typing import List
+from dataclasses import dataclass
+
+import pandas as pd
+import github
+
+from mindsdb.integrations.utilities.sql_utils import (FilterCondition, FilterOperator, SortColumn)
+from mindsdb.integrations.libs.api_handler import APIResource
+
+
+@dataclass
+class Type:
+    name: str
+    sub_type: str = None
+    optional: bool = False
+
+
+@dataclass
+class GHMethod:
+    name: str
+    table_name: str
+    params: dict
+    output: Type
+
+
+def parse_annotations(annotations):
+    '''
+    Parse string annotation, and extract type, input examples:
+    - Milestone | Opt[str]
+    - PaginatedList[Issue]
+    '''
+    type_name, sub_type = None, None
+    if not isinstance(annotations, str):
+
+        return Type(getattr(annotations, '__name__', None))
+    for item in annotations.split('|'):
+        item = item.strip()
+        if item is None:
+            continue
+        if '[' in item:
+            type_name = item[: item.find('[')]
+            item2 = item[item.find('[') + 1: item.rfind(']')]
+            if type_name == 'Opt':
+                inner_type = parse_annotations(item2)
+                inner_type.optional = Type
+                return inner_type
+            if type_name == 'dict':
+                item2 = item2[item2.find(',') + 1:]
+            sub_type = parse_annotations(item2).name
+        else:
+            type_name = item
+        # get only first type
+        break
+    return Type(type_name, sub_type)
+
+
+def get_properties(cls):
+    # find properties of the class
+
+    properties = {}
+    for prop_name, prop in inspect.getmembers(cls):
+        if prop_name.startswith('_'):
+            continue
+        if not isinstance(prop, property):
+            continue
+        sig2 = inspect.signature(prop.fget)
+
+        properties[prop_name] = parse_annotations(sig2.return_annotation)
+    return properties
+
+
+def get_github_types():
+    # get github types
+    types = {}
+
+    GithubObject = github.GithubObject.GithubObject
+    for module_name, module in inspect.getmembers(github, inspect.ismodule):
+        cls = getattr(module, module_name, None)
+        if cls is None:
+            continue
+        if issubclass(cls, GithubObject):
+
+            # remove inherited props
+            parent_props = []
+            for cls2 in cls.__bases__:
+                parent_props += get_properties(cls2).keys()
+
+            properties = {}
+            for k, v in get_properties(cls).items():
+                if k not in parent_props:
+                    properties[k] = v
+
+            types[module_name] = properties
+    return types
+
+
+def get_github_methods(cls):
+    '''
+    Analyse class in order to find methods which return list of objects.
+    '''
+    methods = []
+
+    for method_name, method in inspect.getmembers(cls, inspect.isfunction):
+        sig = inspect.signature(method)
+
+        return_type = parse_annotations(sig.return_annotation)
+        list_prefix = 'get_'
+        if not (method_name.startswith(list_prefix) and return_type.name == 'PaginatedList'):
+            continue
+
+        table_name = method_name[len(list_prefix):]
+
+        params = {}
+        for param_name, param in sig.parameters.items():
+            params[param_name] = parse_annotations(param.annotation)
+
+        methods.append(GHMethod(
+            name=method_name,
+            table_name=table_name,
+            params=params,
+            output=return_type
+        ))
+    return methods
+
+
+class GHTable(APIResource):
+    def __init__(self, *args, method: GHMethod = None, github_types=None, **kwargs):
+        self.method = method
+        self.github_types = github_types
+
+        self.output_columns = {}
+        if method.output.sub_type in self.github_types:
+            self.output_columns = self.github_types[method.output.sub_type]
+
+        # check params:
+        self.params, self.list_params = [], []
+        for name, param_type in method.params.items():
+            self.params.append(name)
+            if param_type.name == 'list':
+                self.list_params.append(name)
+
+        self._allow_sort = 'sort' in method.params
+
+        super().__init__(*args, **kwargs)
+
+    def repr_value(self, value, type_name):
+        if value is None or type_name in ('bool', 'int', 'float'):
+            return value
+        if type_name in self.github_types:
+            properties = self.github_types[type_name]
+            if 'login' in properties:
+                value = getattr(value, 'login')
+            elif 'url' in properties:
+                value = getattr(value, 'url')
+        return str(value)
+
+    def get_columns(self) -> List[str]:
+        return list(self.output_columns.keys())
+
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ) -> pd.DataFrame:
+
+        if limit is None:
+            limit = 20
+
+        method_kwargs = {}
+        if sort is not None and self._allow_sort:
+            for col in sort:
+                method_kwargs['sort'] = col.column
+                method_kwargs['direction'] = 'asc' if col.ascending else 'desc'
+                sort.applied = True
+                # supported only 1 column
+                break
+
+        if conditions:
+            for condition in conditions:
+                if condition.column not in self.params:
+                    continue
+
+                if condition.column in self.list_params:
+                    if condition.op == FilterOperator.IN:
+                        method_kwargs[condition.column] = condition.value
+                    elif condition.op == FilterOperator.EQUAL:
+                        method_kwargs[condition.column] = [condition]
+                    condition.applied = True
+                else:
+                    method_kwargs[condition.column] = condition.value
+                    condition.applied = True
+
+        connection = self.handler.connect()
+        method = getattr(connection.get_repo(self.handler.repository), self.method.name)
+
+        data = []
+        count = 0
+        for record in method(**method_kwargs):
+            item = {}
+            for name, output_type in self.output_columns.items():
+
+                # workaround to prevent making addition request per property.
+                if name in targets:
+                    # request only if is required
+                    value = getattr(record, name)
+                else:
+                    value = getattr(record, '_' + name).value
+                if value is not None:
+                    if output_type.name == 'list':
+                        value = ",".join([
+                                str(self.repr_value(i, output_type.sub_type))
+                                for i in value
+                        ])
+                    else:
+                        value = self.repr_value(value, output_type.name)
+                item[name] = value
+
+            data.append(item)
+
+            count += 1
+            if limit <= count:
+                break
+
+        return pd.DataFrame(data, columns=self.get_columns())

mindsdb/integrations/handlers/github_handler/github_handler.py

@@ -12,7 +12,7 @@ from mindsdb.integrations.handlers.github_handler.github_tables import (
     GithubMilestonesTable,
     GithubProjectsTable, GithubFilesTable
 )
-
+from mindsdb.integrations.handlers.github_handler.generate_api import get_github_types, get_github_methods, GHTable
 from mindsdb.integrations.libs.api_handler import APIHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
@@ -22,6 +22,7 @@ from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
 
+
 class GithubHandler(APIHandler):
     """The GitHub handler implementation"""
 
@@ -43,16 +44,22 @@ class GithubHandler(APIHandler):
         self.connection = None
         self.is_connected = False
 
+        # custom tables
         self._register_table("issues", GithubIssuesTable(self))
-        self._register_table("pull_requests", GithubPullRequestsTable(self))
-        self._register_table("commits", GithubCommitsTable(self))
-        self._register_table("releases", GithubReleasesTable(self))
-        self._register_table("branches", GithubBranchesTable(self))
-        self._register_table("contributors", GithubContributorsTable(self))
-        self._register_table("milestones", GithubMilestonesTable(self))
-        self._register_table("projects", GithubProjectsTable(self))
         self._register_table("files", GithubFilesTable(self))
 
+        # generated tables
+        github_types = get_github_types()
+
+        # generate tables from repository object
+        for method in get_github_methods(github.Repository.Repository):
+            if method.table_name in self._tables:
+                continue
+
+            table = GHTable(self, github_types=github_types, method=method)
+            self._register_table(method.table_name, table)
+
+
     def connect(self) -> StatusResponse:
         """Set up the connection required by the handler.
 

mindsdb/integrations/handlers/github_handler/requirements.txt

@@ -1 +1 @@
-pygithub
+pygithub==2.6.1

mindsdb/integrations/handlers/jira_handler/__init__.py

@@ -4,6 +4,7 @@ from .__about__ import __version__ as version, __description__ as description
 try:
     from .jira_handler import JiraHandler as Handler
     import_error = None
+
 except Exception as e:
     Handler = None
     import_error = e

mindsdb/integrations/handlers/jira_handler/jira_handler.py

@@ -1,23 +1,15 @@
-from mindsdb.integrations.handlers.jira_handler.jira_table import JiraProjectsTable
 from mindsdb.integrations.libs.api_handler import APIHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
 )
 from mindsdb.utilities import log
-from mindsdb_sql_parser import parse_sql
-
-from atlassian import Jira
-from typing import Optional
-import requests
+from mindsdb.integrations.libs.api_handler_generator import APIResourceGenerator
 
 
 logger = log.getLogger(__name__)
 
-class JiraHandler(APIHandler):
-    """
-    This handler handles connection and execution of the Airtable statements.
-    """
 
+class JiraHandler(APIHandler):
 
     def __init__(self, name=None, **kwargs):
         """
@@ -30,64 +22,38 @@ class JiraHandler(APIHandler):
         super().__init__(name)
         self.connection_data = kwargs.get("connection_data", {})
 
-        self.parser = parse_sql
-        self.dialect = 'jira'
-        self.kwargs = kwargs
         self.connection = None
         self.is_connected = False
 
+        # todo store parsed data in files
+
+        self.api_resource_generator = APIResourceGenerator(
+            "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json",
+            self.connection_data,
+            url_base='/rest/api/3/',
+            options={
+                'offset_param': ['startAt', 'offset'],
+                'total_column': ['totalEntryCount', 'total'],
+                'check_connection_table': 'myself'
+            }
+        )
+
+        resource_tables = self.api_resource_generator.generate_api_resources(self)
 
-        jira_projects_data = JiraProjectsTable(self)
-        self._register_table("project", jira_projects_data)
+        for table_name, resource in resource_tables.items():
+            self._register_table(table_name, resource)
 
     def __del__(self):
         if self.is_connected is True:
             self.disconnect()
 
-    def connect(self) -> StatusResponse:
+    def connect(self):
         """
         Set up the connection required by the handler.
         Returns:
             HandlerStatusResponse
         """
-
-        if self.is_connected is True:
-            return self.connection
-        
-        s = requests.Session()
-        if self.connection_data.get("cloud", False):
-            params = {
-                "cloud": True,
-                "username": self.connection_data['jira_username'],
-                "password": self.connection_data['jira_api_token'],
-                "url": self.connection_data['jira_url'],
-            }   
-        else:
-            params = {
-                "cloud": False,
-                "url": self.connection_data['jira_url'],
-                "session": s
-            }
-
-        s.headers['Authorization'] =  f"Bearer {self.connection_data['jira_api_token']}"
-
-        self.connection = Jira(**params)
-        self.is_connected = True
-
-
-        return self.connection
-
-    def disconnect(self):
-        """
-        Close any existing connections.
-        """
-
-        if self.is_connected is False:
-            return
-
-        self.connection.close()
-        self.is_connected = False
-        return self.is_connected
+        return
 
     def check_connection(self) -> StatusResponse:
         """
@@ -97,37 +63,13 @@ class JiraHandler(APIHandler):
         """
 
         response = StatusResponse(False)
-        need_to_close = self.is_connected is False
-        
+
         try:
-            self.connect()
+            self.api_resource_generator.check_connection()
             response.success = True
         except Exception as e:
             logger.error(f"Error connecting to Jira API: {e}!")
             response.error_message = e
 
         self.is_connected = response.success
-
         return response
-
-    def native_query(self, query: str) -> StatusResponse:
-        """Receive and process a raw query.
-        Parameters
-        ----------
-        query : str
-            query in a native format
-        Returns
-        -------
-        StatusResponse
-            Request status
-        """
-        ast = parse_sql(query)
-        return self.query(ast)
-   
-    def construct_jql(self):
-        """construct jql & returns it to JiraProjectsTable class
-        Returns
-        -------
-        Str
-        """
-        return 'project = ' + str(self.connection_data['project'])