MindsDB 1.3.1__py3-none-any.whl → 25.1.2.0__py3-none-any.whl

mindsdb/integrations/handlers/mendeley_handler/mendeley_tables.py

@@ -0,0 +1,134 @@
+from mindsdb_sql_parser import ast
+import pandas as pd
+from mindsdb.integrations.libs.api_handler import APITable
+from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions
+
+class CatalogSearchTable(APITable):
+
+    def select(self, query: ast.Select) -> pd.DataFrame:
+
+        """The select method implements the mappings from the ast.Select and calls the actual API through the call_mendeley_api.
+        Firstly, it is used to put the parameters specified in the query in a dictionary, which is then used when calling the method call_mendeley_api.
+        If no conditions are specified, an error is raised since the search cannot be conducted.
+
+        Args:
+            query (ast.Select): query used to specify the wanted results 
+        Returns:
+            result (DataFrame): the result of the query
+        """
+       
+        conditions = extract_comparison_conditions(query.where)
+        
+        params = {} 
+
+        #Since there are three different types of search, and each of them takes different parameters, we use the parameters that lead
+        # to the most specific results. For example, in the case of the user specifying the title and the doi of a document, priority is given to
+        # the doi.
+
+        if query.limit is not None:
+            params['limit'] = query.limit.value
+        else:
+            params['limit'] = 30
+
+        for op,arg1,arg2 in conditions:
+
+            if arg1 in ['arxiv', 'doi', 'isbn', 'issn', 'pmid', 'scopus', 'filehash']:
+
+                if op != '=':
+                    raise NotImplementedError
+                params[arg1] = arg2
+
+                result = self.handler.call_mendeley_api(
+                method_name='identifier_search',
+                params=params)
+
+                break
+
+            elif arg1 == 'id':
+                if op != '=':
+                    raise NotImplementedError
+                params['id'] = arg2
+
+                result = self.handler.call_mendeley_api(
+                method_name='get',
+                params=params)
+
+                break
+
+            elif "title" or "author" or"source" or "abstract" or "min_year" or "max_year" or "open_access" or "view" in conditions:
+
+                if arg1 == 'title':
+                    if op != '=':
+                        raise NotImplementedError
+                    params['title'] = arg2
+                
+                elif arg1 == 'author':
+                    if op != '=':
+                        raise NotImplementedError
+                    params['author'] = arg2
+                
+                elif arg1 == 'source':
+                    if op != '=':
+                        raise NotImplementedError
+                    params['source'] = arg2
+                
+                elif arg1 == 'abstract':
+                    if op != '=':
+                        raise NotImplementedError
+                    params['abstract'] = arg2
+                
+                elif arg1 == 'min_year':
+                    params['min_year'] = arg2
+                
+                elif arg1 == 'max_year':
+                    params['max_year'] = arg2
+                
+                elif arg1 == 'open_access':
+                    if op != '=':
+                        raise NotImplementedError
+                    params['open_access'] = arg2
+                
+            
+                result = self.handler.call_mendeley_api(
+                method_name='advanced_search',
+                params=params)
+        
+        if conditions==[]:
+            raise ValueError('Please give input for the search to be conducted.')
+        
+
+        columns = []
+
+        for target in query.targets:
+            if isinstance(target, ast.Star):
+                columns = self.get_columns()
+                break
+            elif isinstance(target, ast.Identifier):
+                columns.append(target.parts[-1])
+            else:
+                raise ValueError(f"Unknown query target {type(target)}")
+        
+
+        return result[columns]
+    
+    def get_columns(self):
+        """ get_columns method returns the columns returned by the API"""
+        return [
+
+            'title',
+            'type',
+            'source',
+            'year',
+            'pmid',
+            'sgr',
+            'issn',
+            'scopus',
+            'doi',
+            'pui',
+            'authors',
+            'keywords',
+            'link',
+            'id'
+        ]
+
+        

mindsdb/integrations/handlers/mendeley_handler/requirements.txt

@@ -0,0 +1 @@
+mendeley

mindsdb/integrations/handlers/mendeley_handler/tests/test_mendeley_handler.py

@@ -0,0 +1,108 @@
+import unittest
+
+from mindsdb.integrations.handlers.mendeley_handler.mendeley_handler import MendeleyHandler
+from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
+
+class MendeleyHandlerTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.kwargs = {
+            "connection_data":{"client_id":15253,"client_secret":"BxmSvbrRW5iYEIQR"}
+        }
+        cls.handler = MendeleyHandler("test_mendeley_handler",**cls.kwargs)
+
+    def test_0_connect(self):
+        self.handler.connect()
+
+    def test_1_check_connection(self):
+        self.handler.check_connection()
+
+    def test_2_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE doi='10.1111/joim.13091'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+
+    def test_3_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE id='af1a0408-7409-3a8b-ad91-8accd4f8849a'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_4_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE title='The American Mineralogist crystal structure database'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_5_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE id='8e86b541-84fd-30ef-9eed-e8c9af847ca3' AND doi='10.1093/ajcn/77.1.71'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_6_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE issn='15570878'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_7_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE source='American Journal of Clinical Nutrition'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_8_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE source='American Journal of Clinical Nutrition'AND id='4eeda257-8db4-3dad-80c8-6912356d3887'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_9_select(self):
+        query = "SELECT * FROM catalog_search_data WHERE source='American Journal of Clinical Nutrition'AND max_year='2020'"
+        result = self.handler.native_query(query)
+        assert result.type is RESPONSE_TYPE.TABLE
+    
+    def test_10_call_mendeley_api_invalid_method_name(self):
+        with self.assertRaises(NotImplementedError):
+            res = self.handler.call_mendeley_api("method1",None)
+
+    def test_11_select_invalid_condition_name(self):
+        with self.assertRaises(NotImplementedError):
+            query = "SELECT * FROM catalog_search_data WHERE name='American Journal of Clinical Nutrition'"
+            result = self.handler.native_query(query)
+
+    def test_12_select_invalid_operator(self):
+        with self.assertRaises(NotImplementedError):
+            query = "SELECT * FROM catalog_search_data WHERE source>'American Journal of Clinical Nutrition'AND max_year='2020' "
+            result = self.handler.native_query(query)
+    
+    def test_13_select_invalid_column_name(self):
+        with self.assertRaises(KeyError):
+            query = "SELECT name FROM catalog_search_data WHERE source='American Journal of Clinical Nutrition'AND max_year='2020' "
+            result = self.handler.native_query(query)
+    
+    def test_14_get_columns(self):
+        columns = self.handler.catalog_search_data.get_columns()
+
+        expected_columns = [
+
+            'title',
+            'type',
+            'source',
+            'year',
+            'pmid',
+            'sgr',
+            'issn',
+            'scopus',
+            'doi',
+            'pui',
+            'authors',
+            'keywords',
+            'link',
+            'id'
+        ]
+
+        self.assertListEqual(columns, expected_columns)
+    
+    def test_15_select_invalid_condition_name(self):
+        with self.assertRaises(ValueError):
+            query = "SELECT * FROM catalog_search_data"
+            result = self.handler.native_query(query)
+
+if __name__ == "__main__":
+    unittest.main()

mindsdb/integrations/handlers/merlion_handler/__about__.py

@@ -0,0 +1,9 @@
+__title__ = 'MindsDB Merlion handler'
+__package_name__ = 'mindsdb_merlion_handler'
+__version__ = '0.0.1'
+__description__ = "MindsDB handler for Merlion"
+__author__ = 'Franklin Zhi'
+__github__ = 'https://github.com/mindsdb/mindsdb'
+__pypi__ = 'https://pypi.org/project/mindsdb/'
+__license__ = 'MIT'
+__copyright__ = 'Copyright 2022- mindsdb'

mindsdb/integrations/handlers/merlion_handler/__init__.py

@@ -0,0 +1,19 @@
+from mindsdb.integrations.libs.const import HANDLER_TYPE
+
+from .__about__ import __version__ as version, __description__ as description
+try:
+    from .merlion_handler import MerlionHandler as Handler
+    import_error = None
+except Exception as e:
+    Handler = None
+    import_error = e
+
+title = 'Merlion'
+name = 'merlion'
+type = HANDLER_TYPE.ML
+icon_path = 'icon.svg'
+permanent = False
+
+__all__ = [
+    'Handler', 'version', 'name', 'type', 'title', 'description', 'import_error', 'icon_path'
+]

mindsdb/integrations/handlers/merlion_handler/adapters.py

@@ -0,0 +1,182 @@
+from merlion.models.anomaly.forecast_based.prophet import ProphetDetectorConfig, ProphetDetector
+from merlion.models.anomaly.isolation_forest import IsolationForestConfig, IsolationForest
+from merlion.models.anomaly.windstats import WindStatsConfig, WindStats
+from merlion.models.automl.autoprophet import AutoProphetConfig, AutoProphet
+from merlion.models.automl.autosarima import AutoSarimaConfig, AutoSarima
+from merlion.models.forecast.smoother import MSESConfig, MSES
+from merlion.post_process.threshold import AggregateAlarms
+from merlion.transform.moving_average import DifferenceTransform
+from scipy.stats import norm
+
+from merlion.models.defaults import DefaultDetectorConfig, DefaultDetector, DefaultForecaster, DefaultForecasterConfig
+from merlion.utils import TimeSeries
+
+from enum import Enum
+import pandas as pd
+
+
+class MerlionArguments(Enum):
+    target_seq_index = "target_seq_index"
+    max_forecast_steps = "max_forecast_steps"
+    max_backstep = "max_backstep"
+    wind_sz = "wind_sz"
+    alm_threshold = "alm_threshold"
+    maxiter = "maxiter"
+
+
+class BaseMerlionForecastAdapter:
+    TARGET_SEQ_INDEX = 0
+    DEFAULT_MAX_FORECAST_STEPS = 100
+    DEFAULT_MAX_BACKSTEP = 60
+    DEFAULT_MAXITER = 5
+
+    def __init__(self, **kwargs):
+        self.max_forecast_steps = kwargs.get(MerlionArguments.max_forecast_steps.value, self.DEFAULT_MAX_FORECAST_STEPS)
+        self.max_backstep = kwargs.get(MerlionArguments.max_backstep.value, self.DEFAULT_MAX_BACKSTEP)
+        self.maxiter = kwargs.get(MerlionArguments.maxiter.value, self.DEFAULT_MAXITER)
+        self.model = None
+
+    def to_bytes(self) -> bytes:
+        return self.model.to_bytes()
+
+    def initialize_model(self, bytes):
+        self.model = self.model.from_bytes(bytes)
+
+    def to_train_dataframe(self, df: pd.DataFrame, target: str) -> pd.DataFrame:
+        columns = list(df.columns.values)
+        columns.remove(target)
+        columns.insert(self.TARGET_SEQ_INDEX, target)
+        return df[columns]
+
+    def train(self, df: pd.DataFrame, target: str):
+        df = self.to_train_dataframe(df=df, target=target)
+        train_data = TimeSeries.from_pd(df)
+        self.model.train(train_data=train_data)
+
+    def predict(self, df: pd.DataFrame, target: str) -> pd.DataFrame:
+        forecast_step = self.max_forecast_steps
+        df = df[df.index <= self.model.last_train_time + self.model.timedelta * forecast_step]
+        if len(list(df.columns.values)) == 0:
+            df.loc[:, target] = 0
+        predict_data = TimeSeries.from_pd(df)
+        predict_pred, predict_err = self.model.forecast(time_stamps=predict_data.time_stamps)
+        return self.__prepare_forecast_return(target=target, pred_ts=predict_pred, err_ts=predict_err)
+
+    def __prepare_forecast_return(self, target: str, pred_ts: TimeSeries, err_ts: TimeSeries) -> pd.DataFrame:
+        pred_df: pd.DataFrame = pred_ts.to_pd()
+        if err_ts is None:
+            err_df = None
+        else:
+            err_df = err_ts.to_pd()
+
+        if err_df is None or target in list(err_df.columns.values):  # error and predict sometimes are same
+            std = pred_df[target].std()
+            pred_df[f"{target}__upper"] = pred_df[f"{target}"] + std * norm.ppf(0.975)
+            pred_df[f"{target}__lower"] = pred_df[f"{target}"] + std * norm.ppf(0.025)
+        else:
+            pred_df = pred_df.join(err_df, how="left")
+            pred_df[f"{target}__upper"] = pred_df[f"{target}"] + norm.ppf(0.975) * err_df[f"{target}_err"]
+            pred_df[f"{target}__lower"] = pred_df[f"{target}"] + norm.ppf(0.025) * err_df[f"{target}_err"]
+            pred_df.drop(columns=[f"{target}_err"], inplace=True)
+        return pred_df
+
+
+class DefaultForecasterAdapter(BaseMerlionForecastAdapter):
+    # DefaultForecaster
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.html#
+    #               merlion.models.defaults.DefaultForecaster
+    def __init__(self, **kwargs):
+        super(DefaultForecasterAdapter, self).__init__(**kwargs)
+        self.model = DefaultForecaster(DefaultForecasterConfig(max_forecast_steps=self.max_forecast_steps,
+                                                               target_seq_index=self.TARGET_SEQ_INDEX))
+
+
+class SarimaForecasterAdapter(BaseMerlionForecastAdapter):
+    # AutoSarima
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.automl.html#
+    #               module-merlion.models.automl.autosarima
+    def __init__(self, **kwargs):
+        super(SarimaForecasterAdapter, self).__init__(**kwargs)
+        config = AutoSarimaConfig(auto_pqPQ=True, auto_d=True, auto_D=True, auto_seasonality=True,
+                                  approximation=True, maxiter=5)
+        self.model = AutoSarima(config)
+
+
+class ProphetForecasterAdapter(BaseMerlionForecastAdapter):
+    # AutoProphet
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.automl.html#
+    #               module-merlion.models.automl.autoprophet
+    def __init__(self, **kwargs):
+        super(ProphetForecasterAdapter, self).__init__(**kwargs)
+        self.model = AutoProphet(AutoProphetConfig(max_forecast_steps=self.max_forecast_steps))
+
+
+class MSESForecasterAdapter(BaseMerlionForecastAdapter):
+    # MSES
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.forecast.html#
+    #               merlion.models.forecast.smoother.MSES
+    def __init__(self, **kwargs):
+        super(MSESForecasterAdapter, self).__init__(**kwargs)
+        self.model = MSES(MSESConfig(max_forecast_steps=self.max_forecast_steps,
+                                     target_seq_index=self.TARGET_SEQ_INDEX,
+                                     max_backstep=self.max_backstep))
+
+
+class BaseMerlineDetectorAdapter(BaseMerlionForecastAdapter):
+    DEFAULT_WIND_SZ = 60
+    DEFAULT_ALM_THRESHOLD = 4
+
+    def __init__(self, **kwargs):
+        super(BaseMerlineDetectorAdapter, self).__init__(**kwargs)
+        self.wind_sz = kwargs.get(MerlionArguments.wind_sz.value, self.DEFAULT_WIND_SZ)
+        self.alm_threshold = kwargs.get(MerlionArguments.alm_threshold.value, self.DEFAULT_ALM_THRESHOLD)
+
+    def train(self, df: pd.DataFrame, target: str):
+        df = self.to_train_dataframe(df=df, target=target)
+        train_data = TimeSeries.from_pd(df[target])
+        self.model.train(train_data=train_data)
+
+    def predict(self, df: pd.DataFrame, target: str) -> pd.DataFrame:
+        predict_data = TimeSeries.from_pd(df[target])
+        predict_label = self.model.get_anomaly_label(time_series=predict_data)
+        pred_df = predict_label.to_pd()
+        pred_df[f"{target}__anomaly_score"] = pred_df["anom_score"]
+        return pred_df
+
+
+class DefaultDetectorAdapter(BaseMerlineDetectorAdapter):
+    # DefaultDetector
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.html#
+    #               merlion.models.defaults.DefaultDetector
+    def __init__(self, **kwargs):
+        super(DefaultDetectorAdapter, self).__init__(**kwargs)
+        self.model = DefaultDetector(DefaultDetectorConfig())
+
+
+class IsolationForestDetectorAdapter(BaseMerlineDetectorAdapter):
+    # IsolationForest
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.anomaly.html#
+    #               merlion.models.anomaly.isolation_forest.IsolationForest
+    def __init__(self, **kwargs):
+        super(IsolationForestDetectorAdapter, self).__init__(**kwargs)
+        self.model = IsolationForest(IsolationForestConfig())
+
+
+class WindStatsDetectorAdapter(BaseMerlineDetectorAdapter):
+    # WindStats
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.anomaly.html#
+    #               merlion.models.anomaly.windstats.WindStats
+    def __init__(self, **kwargs):
+        super(WindStatsDetectorAdapter, self).__init__(**kwargs)
+        config = WindStatsConfig(wind_sz=self.wind_sz, threshold=AggregateAlarms(alm_threshold=self.alm_threshold))
+        self.model = WindStats(config)
+
+
+class ProphetDetectorAdapter(BaseMerlineDetectorAdapter):
+    # ProphetDetector
+    # reference: https://opensource.salesforce.com/Merlion/latest/merlion.models.anomaly.forecast_based.html#
+    #               merlion.models.anomaly.forecast_based.prophet.ProphetDetector
+    def __init__(self, **kwargs):
+        super(ProphetDetectorAdapter, self).__init__(**kwargs)
+        config = ProphetDetectorConfig(transform=DifferenceTransform())
+        self.model = ProphetDetector(config)

mindsdb/integrations/handlers/merlion_handler/icon.svg

@@ -0,0 +1,16 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" viewBox="0 0 40 40" fill="none">
+  <g clip-path="url(#clip0_155_1116)">
+    <path d="M19.7184 38.7924C30.5368 38.7924 39.3068 30.283 39.3068 19.786C39.3068 9.28911 30.5368 0.779675 19.7184 0.779675C8.89999 0.779675 0.129944 9.28911 0.129944 19.786C0.129944 30.283 8.89999 38.7924 19.7184 38.7924Z" fill="#00A1E0"/>
+    <mask id="mask0_155_1116" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="0" y="7" width="37" height="34">
+      <path fill-rule="evenodd" clip-rule="evenodd" d="M20.0235 11.4417C20.0235 11.4417 20.4207 10.8838 21.7737 11.5147C21.7737 11.5147 21.7656 12.1774 20.5095 12.2097C19.2538 12.242 19.7803 12.0643 19.7803 12.0643L20.0235 11.4417ZM17.9467 39.2245C25.439 41.6323 29.6698 38.5471 33.3025 36.8871C35.6138 35.831 37.7388 34.5779 35.3792 31.1777C35.1575 29.6671 33.7057 29.5995 32.7862 24.9271C33.4387 24.9213 33.9351 24.9558 34.6742 24.9876C34.6091 24.7815 34.3758 24.4476 34.1536 24.0871L34.1227 24.1036C33.5747 23.121 33.2112 21.7858 32.9909 20.7477L32.9882 20.7373L32.972 20.6603L32.9711 20.6558C32.7916 19.8086 32.7077 19.1616 32.6973 19.0779L32.6969 19.0777L32.6965 19.0711L32.696 19.0687C32.6534 18.7755 32.6264 18.5059 32.6112 18.2739L34.1025 18.596C32.8374 14.707 33.5209 12.5164 30.6913 10.0611L30.9893 8.85596C29.1 9.08151 26.787 7.443 23.4167 8.10822C22.2597 8.71257 22.0748 8.3103 20.7191 9.81375C19.844 10.7846 18.7871 10.5273 18.464 11.9617C17.0433 12.0408 15.3754 12.1709 15.4899 13.7133C15.5301 14.2581 15.597 15.601 15.7676 15.9585C16.1419 16.741 16.6059 16.3804 17.4053 16.6541C17.5382 16.4955 17.6724 16.355 17.8083 16.2307L18.179 16.8134C18.179 16.8134 18.3469 16.1678 18.8836 15.6168L18.9173 15.5536C21.2424 14.7747 23.12 17.8658 19.8758 18.2065L19.1075 18.2021C18.6449 17.8483 18.5084 17.3227 18.5084 17.3227L18.241 17.743L17.3753 18.1917C17.3641 19.1044 17.0321 19.29 17.2074 20.2085C17.9169 20.5158 18.7068 20.5475 19.677 20.5756C15.9385 28.7224 13.1319 21.1027 17.3757 29.7999C18.9213 33.0907 18.7369 36.0187 18.1566 38.4083M15.363 17.8803C15.363 17.8803 5.81092 18.9482 0.00170898 25.2619C0.94665 27.1492 0.797835 27.9135 2.06482 28.3915C7.57434 20.6125 15.2857 17.8255 15.0945 17.9552" fill="white"/>
+    </mask>
+    <g mask="url(#mask0_155_1116)">
+      <path d="M19.7178 38.7933C30.5362 38.7933 39.3062 30.2839 39.3062 19.7869C39.3062 9.29002 30.5362 0.780586 19.7178 0.780586C8.89938 0.780586 0.129333 9.29002 0.129333 19.7869C0.129333 30.2839 8.89938 38.7933 19.7178 38.7933Z" fill="white"/>
+    </g>
+  </g>
+  <defs>
+    <clipPath id="clip0_155_1116">
+      <rect width="40" height="40" fill="white"/>
+    </clipPath>
+  </defs>
+</svg>