MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/integrations/handlers/twitter_handler/twitter_handler.py

@@ -0,0 +1,471 @@
+import re
+import os
+import datetime as dt
+import ast
+import time
+from collections import defaultdict
+import pytz
+import io
+import requests
+
+import pandas as pd
+import tweepy
+
+from mindsdb.utilities import log
+from mindsdb.utilities.config import Config
+
+from mindsdb_sql_parser import ast
+
+from mindsdb.integrations.libs.api_handler import APIHandler, APITable, FuncParser
+from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions
+from mindsdb.integrations.utilities.date_utils import parse_utc_date
+
+from mindsdb.integrations.libs.response import (
+    HandlerStatusResponse as StatusResponse,
+    HandlerResponse as Response,
+    RESPONSE_TYPE
+)
+
+logger = log.getLogger(__name__)
+
+class TweetsTable(APITable):
+
+    def select(self, query: ast.Select) -> Response:
+
+        conditions = extract_comparison_conditions(query.where)
+
+        params = {}
+        filters = []
+        for op, arg1, arg2 in conditions:
+
+            if op == 'or':
+                raise NotImplementedError(f'OR is not supported')
+            if arg1 == 'created_at':
+                date = parse_utc_date(arg2)
+                if op == '>':
+                    # "tweets/search/recent" doesn't accept dates earlier than 7 days
+                    if (dt.datetime.now(dt.timezone.utc) - date).days > 7:
+                        # skip this condition
+                        continue
+                    params['start_time'] = date
+                elif op == '<':
+                    params['end_time'] = date
+                else:
+                    raise NotImplementedError
+
+            elif arg1 == 'query':
+                if op == '=':
+                    params[arg1] = arg2
+                else:
+                    NotImplementedError(f'Unknown op: {op}')
+
+            elif arg1 == 'id':
+                if op == '>':
+                    params['since_id'] = arg2
+                elif op == '>=':
+                    raise NotImplementedError("Please use 'id > value'")
+                elif op == '<':
+                    params['until_id'] = arg2
+                elif op == '<=':
+                    raise NotImplementedError("Please use 'id < value'")
+                else:
+                    NotImplementedError('Search with "id=" is not implemented')
+
+            else:
+                filters.append([op, arg1, arg2])
+
+        if query.limit is not None:
+            params['max_results'] = query.limit.value
+
+        params['expansions'] = ['author_id', 'in_reply_to_user_id']
+        params['tweet_fields'] = ['created_at', 'conversation_id', 'referenced_tweets']
+        params['user_fields'] = ['name', 'username']
+
+        if 'query' not in params:
+            # search not works without query, use 'mindsdb'
+            params['query'] = 'mindsdb'
+
+        result = self.handler.call_twitter_api(
+            method_name='search_recent_tweets',
+            params=params,
+            filters=filters
+        )
+
+        # filter targets
+        columns = []
+        for target in query.targets:
+            if isinstance(target, ast.Star):
+                columns = []
+                break
+            elif isinstance(target, ast.Identifier):
+                columns.append(target.parts[-1])
+            else:
+                raise NotImplementedError
+
+        if len(columns) == 0:
+            columns = self.get_columns()
+
+        # columns to lower case
+        columns = [name.lower() for name in columns]
+
+        if len(result) == 0:
+            result = pd.DataFrame([], columns=columns)
+        else:
+            # add absent columns
+            for col in set(columns) & set(result.columns) ^ set(columns):
+                result[col] = None
+
+            # filter by columns
+            result = result[columns]
+        return result
+
+    def get_columns(self):
+        return [
+            'id',
+            'created_at',
+            'text',
+            'edit_history_tweet_ids',
+            'author_id',
+            'author_name',
+            'author_username',
+            'conversation_id',
+            'in_reply_to_tweet_id',
+            'in_retweeted_to_tweet_id',
+            'in_quote_to_tweet_id',
+            'in_reply_to_user_id',
+            'in_reply_to_user_name',
+            'in_reply_to_user_username',
+        ]
+
+    def insert(self, query: ast.Insert):
+        # https://docs.tweepy.org/en/stable/client.html#tweepy.Client.create_tweet
+        columns = [col.name for col in query.columns]
+
+        insert_params = ('consumer_key', 'consumer_secret', 'access_token', 'access_token_secret')
+        for p in insert_params:
+            if p not in self.handler.connection_args:
+                raise Exception(f'To insert data into Twitter, you need to provide the following parameters when connecting it to MindsDB: {insert_params}')  # noqa
+
+        for row in query.values:
+            params = dict(zip(columns, row))
+
+            # split long text over 280 symbols
+            max_text_len = 280
+            text = params['text']
+
+            # Post image if column media_url is provided, only do this on last tweet
+            media_ids = None
+            if 'media_url' in params:
+                media_url = params.pop('media_url')
+
+                # create an in memory file
+                resp = requests.get(media_url)
+                img = io.BytesIO(resp.content)
+
+                # upload media to twitter
+                api_v1 = self.handler.create_connection(api_version=1)
+                content_type = resp.headers['Content-Type']
+                file_type = content_type.split('/')[-1]
+                media = api_v1.media_upload(filename="img.{file_type}".format(file_type=file_type), file=img)
+
+                media_ids = [media.media_id]
+
+
+            words = re.split('( )', text)
+
+            messages = []
+
+            text2 = ''
+            pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
+            for word in words:
+                # replace the links in word to string with the length as twitter short url (23)
+                word2 = re.sub(pattern, '-' * 23, word)
+                if len(text2) + len(word2) > max_text_len - 3 - 7:  # 3 is for ..., 7 is for (10/11)
+                    messages.append(text2.strip())
+
+                    text2 = ''
+                text2 += word
+
+            # the last message
+            if text2.strip() != '':
+                messages.append(text2.strip())
+
+            len_messages = len(messages)
+            for i, text in enumerate(messages):
+                if i < len_messages - 1:
+                    text += '...'
+                else:
+                    text += ' '
+                    # publish media with the last message
+                    if media_ids is not None:
+                        params['media_ids'] = media_ids
+
+                text += f'({i + 1}/{len_messages})'
+
+                params['text'] = text
+                ret = self.handler.call_twitter_api('create_tweet', params)
+                inserted_id = ret.id[0]
+                params['in_reply_to_tweet_id'] = inserted_id
+
+
+class TwitterHandler(APIHandler):
+    """A class for handling connections and interactions with the Twitter API.
+
+    Attributes:
+        bearer_token (str): The consumer key for the Twitter app.
+        api (tweepy.API): The `tweepy.API` object for interacting with the Twitter API.
+
+    """
+
+    def __init__(self, name=None, **kwargs):
+        super().__init__(name)
+
+        args = kwargs.get('connection_data', {})
+
+        self.connection_args = {}
+        handler_config = Config().get('twitter_handler', {})
+        for k in ['bearer_token', 'consumer_key', 'consumer_secret',
+                  'access_token', 'access_token_secret', 'wait_on_rate_limit']:
+            if k in args:
+                self.connection_args[k] = args[k]
+            elif f'TWITTER_{k.upper()}' in os.environ:
+                self.connection_args[k] = os.environ[f'TWITTER_{k.upper()}']
+            elif k in handler_config:
+                self.connection_args[k] = handler_config[k]
+
+        self.api = None
+        self.is_connected = False
+
+        tweets = TweetsTable(self)
+        self._register_table('tweets', tweets)
+
+    def create_connection(self, api_version=2):
+        if api_version == 1:
+            auth = tweepy.OAuthHandler(
+                self.connection_args['consumer_key'],
+                self.connection_args['consumer_secret']
+            )
+            auth.set_access_token(
+                self.connection_args['access_token'],
+                self.connection_args['access_token_secret']
+            )
+            return tweepy.API(auth)
+
+        return tweepy.Client(**self.connection_args)
+
+    def connect(self, api_version=2):
+        """Authenticate with the Twitter API using the API keys and secrets stored in the `consumer_key`, `consumer_secret`, `access_token`, and `access_token_secret` attributes."""  # noqa
+
+        if self.is_connected is True:
+            return self.api
+
+        self.api = self.create_connection()
+
+        self.is_connected = True
+        return self.api
+
+    def check_connection(self) -> StatusResponse:
+
+        response = StatusResponse(False)
+
+        try:
+            api = self.connect()
+
+            # call get_user with unknown id.
+            #   it raises an error in case if auth is not success and returns not-found otherwise
+            #   api.get_me() is not exposed for OAuth 2.0 App-only authorisation
+            api.get_user(id=1)
+            response.success = True
+
+        except tweepy.Unauthorized as e:
+            response.error_message = f'Error connecting to Twitter api: {e}. Check bearer_token'
+            logger.error(response.error_message)
+
+        if response.success is True and len(self.connection_args) > 1:
+            # not only bearer_token, check read-write mode (OAuth 2.0 Authorization Code with PKCE)
+            try:
+                api = self.connect()
+
+                api.get_me()
+
+            except tweepy.Unauthorized as e:
+                keys = 'consumer_key', 'consumer_secret', 'access_token', 'access_token_secret'
+                response.error_message = f'Error connecting to Twitter api: {e}. Check' + ', '.join(keys)
+                logger.error(response.error_message)
+
+                response.success = False
+
+        if response.success is False and self.is_connected is True:
+            self.is_connected = False
+
+        return response
+
+    def native_query(self, query_string: str = None):
+        method_name, params = FuncParser().from_string(query_string)
+
+        df = self.call_twitter_api(method_name, params)
+
+        return Response(
+            RESPONSE_TYPE.TABLE,
+            data_frame=df
+        )
+
+    def _apply_filters(self, data, filters):
+        if not filters:
+            return data
+
+        data2 = []
+        for row in data:
+            add = False
+            for op, key, value in filters:
+                value2 = row.get(key)
+                if isinstance(value, int):
+                    # twitter returns ids as string
+                    value = str(value)
+
+                if op in ('!=', '<>'):
+                    if value == value2:
+                        break
+                elif op in ('==', '='):
+                    if value != value2:
+                        break
+                elif op == 'in':
+                    if not isinstance(value, list):
+                        value = [value]
+                    if value2 not in value:
+                        break
+                elif op == 'not in':
+                    if not isinstance(value, list):
+                        value = [value]
+                    if value2 in value:
+                        break
+                else:
+                    raise NotImplementedError(f'Unknown filter: {op}')
+                # only if there wasn't breaks
+                add = True
+            if add:
+                data2.append(row)
+        return data2
+
+    def call_twitter_api(self, method_name: str = None, params: dict = None, filters: list = None):
+
+        # method > table > columns
+        expansions_map = {
+            'search_recent_tweets': {
+                'users': ['author_id', 'in_reply_to_user_id'],
+            },
+            'search_all_tweets': {
+                'users': ['author_id'],
+            },
+        }
+
+        api = self.connect()
+        method = getattr(api, method_name)
+
+        # pagination handle
+
+        count_results = None
+        if 'max_results' in params:
+            count_results = params['max_results']
+
+        data = []
+        includes = defaultdict(list)
+
+        max_page_size = 100
+        min_page_size = 10
+        left = None
+
+        limit_exec_time = time.time() + 60
+
+        if filters:
+            # if we have filters: do big page requests
+            params['max_results'] = max_page_size
+
+        while True:
+            if time.time() > limit_exec_time:
+                raise RuntimeError('Handler request timeout error')
+
+            if count_results is not None:
+                left = count_results - len(data)
+                if left == 0:
+                    break
+                elif left < 0:
+                    # got more results that we need
+                    data = data[:left]
+                    break
+
+                if left > max_page_size:
+                    params['max_results'] = max_page_size
+                elif left < min_page_size:
+                    params['max_results'] = min_page_size
+                else:
+                    params['max_results'] = left
+
+            logger.debug(f'>>>twitter in: {method_name}({params})')
+            resp = method(**params)
+
+            if hasattr(resp, 'includes'):
+                for table, records in resp.includes.items():
+                    includes[table].extend([r.data for r in records])
+
+            if isinstance(resp.data, list):
+                chunk = [r.data for r in resp.data]
+            else:
+                if isinstance(resp.data, dict):
+                    data.append(resp.data)
+                if hasattr(resp.data, 'data') and isinstance(resp.data.data, dict):
+                    data.append(resp.data.data)
+                break
+
+            # unwind columns
+            for row in chunk:
+                if 'referenced_tweets' in row:
+                    refs = row['referenced_tweets']
+                    if isinstance(refs, list) and len(refs) > 0:
+                        if refs[0]['type'] == 'replied_to':
+                            row['in_reply_to_tweet_id'] = refs[0]['id']
+                        if refs[0]['type'] == 'retweeted':
+                            row['in_retweeted_to_tweet_id'] = refs[0]['id']
+                        if refs[0]['type'] == 'quoted':
+                            row['in_quote_to_tweet_id'] = refs[0]['id']
+
+            if filters:
+                chunk = self._apply_filters(chunk, filters)
+
+            # limit output
+            if left is not None:
+                chunk = chunk[:left]
+
+            data.extend(chunk)
+            # next page ?
+            if count_results is not None and hasattr(resp, 'meta') and 'next_token' in resp.meta:
+                params['next_token'] = resp.meta['next_token']
+            else:
+                break
+
+        df = pd.DataFrame(data)
+
+        # enrich
+        expansions = expansions_map.get(method_name)
+        if expansions is not None:
+            for table, records in includes.items():
+                df_ref = pd.DataFrame(records)
+
+                if table not in expansions:
+                    continue
+
+                for col_id in expansions[table]:
+                    col = col_id[:-3]  # cut _id
+                    if col_id not in df.columns:
+                        continue
+
+                    col_map = {
+                        col_ref: f'{col}_{col_ref}'
+                        for col_ref in df_ref.columns
+                    }
+                    df_ref2 = df_ref.rename(columns=col_map)
+                    df_ref2 = df_ref2.drop_duplicates(col_id)
+
+                    df = df.merge(df_ref2, on=col_id, how='left')
+
+        return df

mindsdb/integrations/handlers/unify_handler/__about__.py

@@ -0,0 +1,9 @@
+__title__ = 'MindsDB Unify handler'
+__package_name__ = 'mindsdb_unify_handler'
+__version__ = '0.0.1'
+__description__ = "MindsDB handler for Unify"
+__author__ = 'Janice Herman & Mohamed Abderrahim'
+__github__ = 'https://github.com/mindsdb/mindsdb'
+__pypi__ = 'https://pypi.org/project/mindsdb/'
+__license__ = 'MIT'
+__copyright__ = 'Copyright 2024 - mindsdb'

mindsdb/integrations/handlers/unify_handler/__init__.py

@@ -0,0 +1,22 @@
+from mindsdb.integrations.libs.const import HANDLER_TYPE
+
+from .__about__ import __version__ as version, __description__ as description
+from .creation_args import creation_args
+from .model_using_args import model_using_args
+try:
+    from .unify_handler import UnifyHandler as Handler
+    import_error = None
+except Exception as e:
+    Handler = None
+    import_error = e
+
+title = 'Unify'
+name = 'unify'
+type = HANDLER_TYPE.ML
+icon_path = 'icon.svg'
+permanent = False
+
+__all__ = [
+    'Handler', 'version', 'name', 'type', 'title', 'description',
+    'import_error', 'icon_path', 'creation_args', 'model_using_args'
+]

mindsdb/integrations/handlers/unify_handler/creation_args.py

@@ -0,0 +1,14 @@
+from collections import OrderedDict
+
+from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE
+
+
+creation_args = OrderedDict(
+    unify_api_key={
+        'type': ARG_TYPE.STR,
+        'description': 'Key for Unify API.',
+        'required': False,
+        'label': 'Unify API key',
+        'secret': True
+    }
+)

mindsdb/integrations/handlers/unify_handler/icon.svg

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="130px" height="113px" style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g><path style="opacity:1" fill="#03b728" d="M 65.5,2.5 C 73.1739,2.33365 80.8406,2.50031 88.5,3C 93.8723,6.50224 99.8723,8.6689 106.5,9.5C 107.473,12.13 107.806,14.63 107.5,17C 104.529,17.7943 101.529,18.461 98.5,19C 98.8333,19.3333 99.1667,19.6667 99.5,20C 98.3774,21.0437 97.0441,21.7104 95.5,22C 96.4158,22.2784 97.0825,22.7784 97.5,23.5C 96.1667,23.5 94.8333,23.5 93.5,23.5C 91.0462,22.7729 88.7128,21.7729 86.5,20.5C 86.3893,19.8826 86.056,19.3826 85.5,19C 82.6667,18.8333 79.8333,18.6667 77,18.5C 71.3705,18.4851 65.8705,19.1518 60.5,20.5C 60.5,19.5 60.5,18.5 60.5,17.5C 59.1667,17.5 57.8333,17.5 56.5,17.5C 56.0824,13.7491 57.749,11.5824 61.5,11C 60.8333,10.6667 60.1667,10.3333 59.5,10C 63.6379,9.3516 64.3045,7.68494 61.5,5C 63.0664,4.3918 64.3998,3.55847 65.5,2.5 Z"/></g>
+<g><path style="opacity:1" fill="#25a137" d="M 106.5,9.5 C 99.8723,8.6689 93.8723,6.50224 88.5,3C 80.8406,2.50031 73.1739,2.33365 65.5,2.5C 80.2327,-1.73737 93.8994,0.595967 106.5,9.5 Z"/></g>
+<g><path style="opacity:1" fill="#08b52a" d="M 65.5,2.5 C 64.3998,3.55847 63.0664,4.3918 61.5,5C 64.3045,7.68494 63.6379,9.3516 59.5,10C 60.1667,10.3333 60.8333,10.6667 61.5,11C 57.749,11.5824 56.0824,13.7491 56.5,17.5C 57.8333,17.5 59.1667,17.5 60.5,17.5C 60.5,18.5 60.5,19.5 60.5,20.5C 65.8705,19.1518 71.3705,18.4851 77,18.5C 79.8333,18.6667 82.6667,18.8333 85.5,19C 86.056,19.3826 86.3893,19.8826 86.5,20.5C 79.7305,18.9774 73.0638,19.3108 66.5,21.5C 63.1667,22.5 59.8333,23.5 56.5,24.5C 55.6192,28.7739 54.2859,32.9405 52.5,37C 52.5699,37.7648 52.9033,38.2648 53.5,38.5C 53.5,38.8333 53.5,39.1667 53.5,39.5C 52.0689,39.8807 51.2355,40.8807 51,42.5C 50.4436,44.8848 50.6103,47.2181 51.5,49.5C 50.0527,50.6222 49.0527,52.1222 48.5,54C 52.3605,54.5691 52.3605,55.4024 48.5,56.5C 49.3256,57.451 49.4922,58.451 49,59.5C 47.6667,58.1667 46.3333,58.1667 45,59.5C 44.009,56.4188 42.1757,55.7521 39.5,57.5C 37.7531,56.0193 36.5864,54.0193 36,51.5C 35.3333,45.1667 35.3333,38.8333 36,32.5C 38.3639,28.7814 39.8639,24.7814 40.5,20.5C 46.2003,10.8188 54.5337,4.8188 65.5,2.5 Z"/></g>
+<g><path style="opacity:1" fill="#07b529" d="M 106.5,9.5 C 120.676,19.5181 128.342,33.1848 129.5,50.5C 129.5,52.1667 129.5,53.8333 129.5,55.5C 128.724,63.9333 126.391,71.9333 122.5,79.5C 121.5,79.5 120.5,79.5 119.5,79.5C 118.226,83.5084 116.226,87.8418 113.5,92.5C 106.814,101.178 98.1471,106.844 87.5,109.5C 74.1609,112.132 60.8275,112.132 47.5,109.5C 47.1667,109.5 46.8333,109.5 46.5,109.5C 44.3428,107.588 42.0094,105.921 39.5,104.5C 40.3984,103.842 41.3984,103.176 42.5,102.5C 41.1667,101.833 39.8333,101.167 38.5,100.5C 39.0253,99.8076 39.6919,99.3076 40.5,99C 43.5084,99.037 46.1751,99.537 48.5,100.5C 50.1151,99.846 51.7818,99.346 53.5,99C 53.1667,98.6667 52.8333,98.3333 52.5,98C 53.8333,97.3333 54.8333,96.3333 55.5,95C 63.8333,94.6667 72.1667,94.3333 80.5,94C 81.056,93.6174 81.3893,93.1174 81.5,92.5C 88.9729,89.0339 95.6395,84.3672 101.5,78.5C 105.206,74.7887 108.206,70.4554 110.5,65.5C 111.665,56.5185 111.832,47.5185 111,38.5C 110.683,40.3008 110.016,40.6341 109,39.5C 105.859,33.5026 102.025,28.1692 97.5,23.5C 97.0825,22.7784 96.4158,22.2784 95.5,22C 97.0441,21.7104 98.3774,21.0437 99.5,20C 99.1667,19.6667 98.8333,19.3333 98.5,19C 101.529,18.461 104.529,17.7943 107.5,17C 107.806,14.63 107.473,12.13 106.5,9.5 Z"/></g>
+<g><path style="opacity:1" fill="#259a35" d="M 66.5,21.5 C 59.5566,25.1172 55.2232,30.7839 53.5,38.5C 52.9033,38.2648 52.5699,37.7648 52.5,37C 54.2859,32.9405 55.6192,28.7739 56.5,24.5C 59.8333,23.5 63.1667,22.5 66.5,21.5 Z"/></g>
+<g><path style="opacity:0.887" fill="#339d45" d="M -0.5,33.5 C 6.16667,33.5 12.8333,33.5 19.5,33.5C 18.6576,38.787 18.3243,44.1203 18.5,49.5C 17.5128,44.6946 17.1795,39.6946 17.5,34.5C 11.8333,34.5 6.16667,34.5 0.5,34.5C 0.0963569,45.3138 1.09636,55.9805 3.5,66.5C 2.01312,62.8835 0.679783,59.2169 -0.5,55.5C -0.5,48.1667 -0.5,40.8333 -0.5,33.5 Z"/></g>
+<g><path style="opacity:1" fill="#03b727" d="M 18.5,49.5 C 18.5,49.8333 18.5,50.1667 18.5,50.5C 19.4118,57.2358 19.4118,63.7358 18.5,70C 16.0937,70.7789 14.427,69.9456 13.5,67.5C 11.443,69.269 9.60966,71.269 8,73.5C 7.68367,70.5043 6.18367,68.5043 3.5,67.5C 3.5,67.1667 3.5,66.8333 3.5,66.5C 1.09636,55.9805 0.0963569,45.3138 0.5,34.5C 6.16667,34.5 11.8333,34.5 17.5,34.5C 17.1795,39.6946 17.5128,44.6946 18.5,49.5 Z"/></g>
+<g><path style="opacity:1" fill="#259836" d="M 84.5,36.5 C 80.1539,36.3343 75.8206,36.501 71.5,37C 69.34,38.332 67.34,39.832 65.5,41.5C 66.6582,38.1783 68.9916,36.0117 72.5,35C 75.8333,34.3333 79.1667,34.3333 82.5,35C 83.4158,35.2784 84.0825,35.7784 84.5,36.5 Z"/></g>
+<g><path style="opacity:1" fill="#04b728" d="M 84.5,36.5 C 84.8333,36.5 85.1667,36.5 85.5,36.5C 85.7031,38.0443 86.3698,39.3777 87.5,40.5C 86.1091,42.9358 86.6091,43.6025 89,42.5C 90.1581,42.9981 90.9915,43.8315 91.5,45C 92.3276,48.4647 92.9942,51.9647 93.5,55.5C 92.551,60.0096 91.2177,64.343 89.5,68.5C 87.7607,71.2427 85.4274,73.2427 82.5,74.5C 80.5,74.5 78.5,74.5 76.5,74.5C 76.3893,73.8826 76.056,73.3826 75.5,73C 77.6193,72.9024 79.286,72.0691 80.5,70.5C 74.8632,70.4556 69.8632,68.7889 65.5,65.5C 64.6143,64.675 64.281,63.675 64.5,62.5C 63.325,62.719 62.325,62.3857 61.5,61.5C 60.4457,59.7988 59.4457,58.1321 58.5,56.5C 59.7095,56.9321 60.7095,57.5987 61.5,58.5C 61.6107,59.1174 61.944,59.6174 62.5,60C 65.8333,60.6667 69.1667,60.6667 72.5,60C 73.9733,59.2904 74.6399,58.1238 74.5,56.5C 65.9875,55.202 62.9875,50.202 65.5,41.5C 67.34,39.832 69.34,38.332 71.5,37C 75.8206,36.501 80.1539,36.3343 84.5,36.5 Z"/></g>
+<g><path style="opacity:1" fill="#209e32" d="M 93.5,23.5 C 94.8333,23.5 96.1667,23.5 97.5,23.5C 102.025,28.1692 105.859,33.5026 109,39.5C 110.016,40.6341 110.683,40.3008 111,38.5C 111.832,47.5185 111.665,56.5185 110.5,65.5C 108.206,70.4554 105.206,74.7887 101.5,78.5C 104.658,73.2173 107.158,67.5507 109,61.5C 111.776,45.4087 106.609,32.7421 93.5,23.5 Z"/></g>
+<g><path style="opacity:1" fill="#21a034" d="M 85.5,36.5 C 94.4333,42.1891 97.2666,50.1891 94,60.5C 93.1404,63.6328 91.6404,66.2995 89.5,68.5C 91.2177,64.343 92.551,60.0096 93.5,55.5C 92.9942,51.9647 92.3276,48.4647 91.5,45C 90.9915,43.8315 90.1581,42.9981 89,42.5C 86.6091,43.6025 86.1091,42.9358 87.5,40.5C 86.3698,39.3777 85.7031,38.0443 85.5,36.5 Z"/></g>
+<g><path style="opacity:1" fill="#259d37" d="M 40.5,20.5 C 39.8639,24.7814 38.3639,28.7814 36,32.5C 35.3333,38.8333 35.3333,45.1667 36,51.5C 36.5864,54.0193 37.7531,56.0193 39.5,57.5C 38.8872,60.9694 40.2206,62.9694 43.5,63.5C 43.3433,64.8734 43.51,66.2068 44,67.5C 45.3756,68.7051 46.5423,70.0384 47.5,71.5C 40.3108,66.8287 35.8108,60.162 34,51.5C 31.9237,40.1555 34.0903,29.8222 40.5,20.5 Z"/></g>
+<g><path style="opacity:1" fill="#289937" d="M 65.5,41.5 C 62.9875,50.202 65.9875,55.202 74.5,56.5C 74.6399,58.1238 73.9733,59.2904 72.5,60C 69.1667,60.6667 65.8333,60.6667 62.5,60C 61.944,59.6174 61.6107,59.1174 61.5,58.5C 65.5738,59.9904 69.5738,59.8238 73.5,58C 67.0371,57.7396 63.3704,54.4063 62.5,48C 63.1319,45.5848 64.1319,43.4181 65.5,41.5 Z"/></g>
+<g><path style="opacity:1" fill="#1e9a33" d="M 53.5,39.5 C 51.7157,46.15 53.3823,51.8166 58.5,56.5C 54.4891,55.6924 52.1557,53.359 51.5,49.5C 50.6103,47.2181 50.4436,44.8848 51,42.5C 51.2355,40.8807 52.0689,39.8807 53.5,39.5 Z"/></g>
+<g><path style="opacity:1" fill="#03b727" d="M 51.5,49.5 C 52.1557,53.359 54.4891,55.6924 58.5,56.5C 59.4457,58.1321 60.4457,59.7988 61.5,61.5C 62.325,62.3857 63.325,62.719 64.5,62.5C 64.281,63.675 64.6143,64.675 65.5,65.5C 69.8632,68.7889 74.8632,70.4556 80.5,70.5C 79.286,72.0691 77.6193,72.9024 75.5,73C 76.056,73.3826 76.3893,73.8826 76.5,74.5C 74.4924,76.9272 71.8257,77.9272 68.5,77.5C 63.4762,76.9402 58.4762,76.2735 53.5,75.5C 51.1184,74.6493 49.1184,73.316 47.5,71.5C 46.5423,70.0384 45.3756,68.7051 44,67.5C 43.51,66.2068 43.3433,64.8734 43.5,63.5C 40.2206,62.9694 38.8872,60.9694 39.5,57.5C 42.1757,55.7521 44.009,56.4188 45,59.5C 46.3333,58.1667 47.6667,58.1667 49,59.5C 49.4922,58.451 49.3256,57.451 48.5,56.5C 52.3605,55.4024 52.3605,54.5691 48.5,54C 49.0527,52.1222 50.0527,50.6222 51.5,49.5 Z"/></g>
+<g><path style="opacity:1" fill="#1f9936" d="M 76.5,74.5 C 78.5,74.5 80.5,74.5 82.5,74.5C 81.9423,75.7899 80.9423,76.6232 79.5,77C 70.5207,80.3593 61.8541,79.8593 53.5,75.5C 58.4762,76.2735 63.4762,76.9402 68.5,77.5C 71.8257,77.9272 74.4924,76.9272 76.5,74.5 Z"/></g>
+<g><path style="opacity:1" fill="#1e9c2e" d="M 122.5,79.5 C 120.662,84.6654 117.662,88.9987 113.5,92.5C 116.226,87.8418 118.226,83.5084 119.5,79.5C 120.5,79.5 121.5,79.5 122.5,79.5 Z"/></g>
+<g><path style="opacity:1" fill="#0ab42a" d="M 18.5,50.5 C 21.513,67.8728 30.8464,80.5394 46.5,88.5C 48.5862,90.8216 51.0862,92.8216 54,94.5C 55.6995,93.5939 57.5328,93.2606 59.5,93.5C 66.8625,93.6354 74.1958,93.3021 81.5,92.5C 81.3893,93.1174 81.056,93.6174 80.5,94C 72.1667,94.3333 63.8333,94.6667 55.5,95C 54.8333,96.3333 53.8333,97.3333 52.5,98C 52.8333,98.3333 53.1667,98.6667 53.5,99C 51.7818,99.346 50.1151,99.846 48.5,100.5C 46.1751,99.537 43.5084,99.037 40.5,99C 39.6919,99.3076 39.0253,99.8076 38.5,100.5C 39.8333,101.167 41.1667,101.833 42.5,102.5C 41.3984,103.176 40.3984,103.842 39.5,104.5C 36.6192,103.475 33.9525,101.975 31.5,100C 30.2068,99.51 28.8734,99.3433 27.5,99.5C 16.108,91.4329 8.10805,80.7663 3.5,67.5C 6.18367,68.5043 7.68367,70.5043 8,73.5C 9.60966,71.269 11.443,69.269 13.5,67.5C 14.427,69.9456 16.0937,70.7789 18.5,70C 19.4118,63.7358 19.4118,57.2358 18.5,50.5 Z"/></g>
+<g><path style="opacity:0.867" fill="#2b963f" d="M 46.5,88.5 C 50.6962,90.7833 55.0295,92.4499 59.5,93.5C 57.5328,93.2606 55.6995,93.5939 54,94.5C 51.0862,92.8216 48.5862,90.8216 46.5,88.5 Z"/></g>
+<g><path style="opacity:0.941" fill="#239c37" d="M 27.5,99.5 C 28.8734,99.3433 30.2068,99.51 31.5,100C 33.9525,101.975 36.6192,103.475 39.5,104.5C 42.0094,105.921 44.3428,107.588 46.5,109.5C 39.3998,107.452 33.0665,104.118 27.5,99.5 Z"/></g>
+<g><path style="opacity:1" fill="#2a9c40" d="M 47.5,109.5 C 60.8275,112.132 74.1609,112.132 87.5,109.5C 85.5,110.5 83.5,111.5 81.5,112.5C 72.5,112.5 63.5,112.5 54.5,112.5C 52.2209,111.349 49.8876,110.349 47.5,109.5 Z"/></g>
+</svg>

mindsdb/integrations/handlers/unify_handler/model_using_args.py

@@ -0,0 +1,5 @@
+model_using_args = {
+    'unify_api_key': {
+        'secret': True
+    }
+}

mindsdb/integrations/handlers/unify_handler/requirements.txt

@@ -0,0 +1 @@
+unifyai==0.9.2

mindsdb/integrations/handlers/unify_handler/tests/test_unify_handler.py

@@ -0,0 +1,60 @@
+import os
+import pytest
+import pandas as pd
+
+from mindsdb_sql_parser import parse_sql
+
+from tests.unit.executor_test_base import BaseExecutorTest
+
+class TestUnify(BaseExecutorTest):
+    
+    def run_sql(self, sql):
+        ret = self.command_executor.execute_command(parse_sql(sql))
+        assert ret.error_code is None
+        if ret.data is not None:
+            return ret.data.to_df()
+
+
+    """
+    Integration tests for the Unify handler.
+    """
+    @pytest.fixture(autouse=True, scope="function")
+    def setup_method(self):
+        """
+        Setup test environment, creating a project
+        """
+        super().setup_method()
+        self.run_sql("create database proj")
+    
+    @pytest.mark.skipif(os.environ.get('UNIFY_API_KEY') is None, reason='Missing API key!')
+    def test_unify_correct_flow(self):
+        self.run_sql(
+        f"""
+        CREATE ML_ENGINE unify_engine
+        FROM unify
+        USING
+            unify_api_key = '{os.environ.get('UNIFY_API_KEY')}';
+        """
+        )
+        
+        self.run_sql(
+        f"""
+        CREATE MODEL proj.test_unify_correct_flow
+        PREDICT output
+        USING
+            engine='unify_engine',
+            model = 'llama-3-8b-chat',
+            provider = 'together-ai',
+            question_column = 'text';
+        """
+        )
+        self.wait_predictor("proj", "test_unify_correct_flow")
+
+        result_df = self.run_sql(
+        """
+            SELECT text, output
+            FROM proj.test_unify_correct_flow
+            WHERE text = 'Hello';
+        """
+        )
+        assert "hello" in result_df["output"].iloc[0].lower()