deepfos 1.1.60__py3-none-any.whl

deepfos/db/clickhouse.py

@@ -0,0 +1,191 @@
+"""数据库连接类"""
+import asyncio
+from typing import Union, List, Iterable, TYPE_CHECKING
+
+import pandas as pd
+from cachetools import TTLCache
+
+from deepfos.api.datatable import ClickHouseAPI
+from deepfos.cache import Manager
+from deepfos.lib.utils import cachedclass, split_dataframe
+from deepfos.lib.decorator import singleton, cached_property
+from deepfos.options import OPTION
+from .dbkits import BaseSqlParser, SyncMeta, T_DataInfo
+from .connector import ClickHouseAPIConnector, ClickHouseDirectAccess
+from .mysql import _AbsAsyncMySQLClient, MySQLConvertor  # noqa
+
+
+__all__ = [
+    'ClickHouseClient',
+    'AsyncClickHouseClient',
+    'ClickHouseConvertor',
+]
+
+
+@singleton
+class SqlParser(BaseSqlParser):
+    api_cls = ClickHouseAPI
+
+    @cached_property
+    def datatable_cls(self):
+        from deepfos.element.datatable import AsyncDataTableClickHouse
+        return AsyncDataTableClickHouse
+
+
+class ClickHouseConvertor(MySQLConvertor):
+    def build_sql(
+        self,
+        columns: str,
+        values_in_line: Iterable[str],
+        tablename: str,
+        updatecol: Iterable[str] = None,
+        **opts
+    ):
+        if updatecol is not None:
+            raise NotImplementedError("`updatecol` is not yet implemented for ClickHouseDB.")
+
+        return super().build_sql(columns, values_in_line, tablename, updatecol, **opts)
+
+
+# -----------------------------------------------------------------------------
+# core
+class APIBasedClient(_AbsAsyncMySQLClient):
+    convertor = ClickHouseConvertor(quote_char='`')
+
+    def __init__(self, version: Union[float, str] = None):  # noqa
+        self.parser = SqlParser()
+        self.connector = ClickHouseAPIConnector(version)
+        self.connector.trxn_execute = self.connector.execute_many
+
+
+class DirectAccessClient:
+    def __init__(self, version: Union[float, str] = None):
+        self.parser = SqlParser()
+        self.connector = ClickHouseDirectAccess(version)
+
+    async def exec_sqls(
+        self,
+        sqls: Union[str, Iterable[str]],
+        table_info: T_DataInfo = None
+    ):
+        """执行多句sql
+
+        Args:
+            sqls: 要执行的sql语句，str或多句sql的list
+            table_info: sql中表名占位符对应的数据表元素信息。
+
+        """
+
+        if isinstance(sqls, str):
+            sqls = [sqls]
+
+        parsed_sql = await self.parser.parse(sqls, table_info)
+        return await self.connector.execute_many(parsed_sql)
+
+    async def query_dfs(
+        self,
+        sqls: Union[str, Iterable[str]],
+        table_info: T_DataInfo = None
+    ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+        """执行sql查询语句
+
+        获取DataFrame格式的二维表
+
+        Args:
+            sqls: 执行的sql，表名可以通过${table_name}占位
+            table_info: sql中表名占位符对应的数据表元素信息。
+
+        Notes:
+            如果执行的sql中没有任何表名占位符，sql将直接执行。
+            如果有占位符, 例如 ``${table1}``，那么要求 ``table_info`` 有
+            key值为 ``table1`` , 对应键值为包含
+            ``elementName, elementType, folderId/path`` 的字典，
+            或者 :class:`DataTableMySQL` 类型，或者 :class:`ElementModel` 类型
+
+        Returns:
+            :class:`DataFrame` 格式的二维数据表
+
+        """
+        if isinstance(sqls, str):
+            sql_list = await self.parser.parse([sqls], table_info)
+            return await self.connector.query_dataframe(sql_list[0])
+        else:
+            sqls = await self.parser.parse(sqls, table_info)
+            return await asyncio.gather(*[
+                self.connector.query_dataframe(sql) for sql in sqls
+            ])
+
+    async def insert_df(
+        self,
+        dataframe: pd.DataFrame,
+        element_name: str = None,
+        table_name: str = None,
+        table_info: T_DataInfo = None,
+        chunksize: int = None,
+    ):
+        """将 :class:`DataFrame` 的插入数据表
+
+        Args:
+            dataframe: 入库数据
+            element_name: 数据表元素名
+            table_name: 数据表的 **实际表名**
+            table_info: 数据表元素信息，使用table
+            chunksize: 单次插库的数据行数
+
+        """
+        if table_name is not None:
+            tbl_name = table_name
+        elif element_name is not None:
+            tbl_name = (await self.parser.parse(["${%s}" % element_name], table_info))[0]
+        else:
+            raise ValueError("Either 'element_name' or 'table_name' must be presented.")
+
+        return await asyncio.gather(*[
+            self.connector.insert_dataframe(tbl_name, df)
+            for df in split_dataframe(dataframe, chunksize)
+        ])
+
+
+if OPTION.general.db_direct_access:
+    ClientBase = DirectAccessClient
+else:
+    ClientBase = APIBasedClient
+
+
+@cachedclass(Manager.create_cache(TTLCache, maxsize=5, ttl=3600))
+class AsyncClickHouseClient(ClientBase):
+    pass
+
+
+@cachedclass(Manager.create_cache(TTLCache, maxsize=5, ttl=3600))
+class ClickHouseClient(ClientBase, metaclass=SyncMeta):
+    synchronize = (
+        'exec_sqls',
+        'query_dfs',
+        'insert_df',
+    )
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def exec_sqls(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ):
+            ...
+
+        def query_dfs(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+            ...
+
+        def insert_df(
+            self,
+            dataframe: pd.DataFrame,
+            element_name: str = None,
+            table_name: str = None,
+            table_info: T_DataInfo = None,
+            chunksize: int = None,
+        ):
+            ...

deepfos/db/connector.py

@@ -0,0 +1,195 @@
+import asyncio
+from abc import ABC, abstractmethod
+from typing import Union, List
+
+import pandas as pd
+
+from deepfos.api.datatable import MySQLAPI, ClickHouseAPI, OracleAPI, SQLServerAPI, KingBaseAPI, GaussAPI, DaMengAPI, \
+    PostgreSQLAPI, DeepEngineAPI, DeepModelAPI, DeepModelKingBaseAPI
+from deepfos.lib.asynchronous import cache_async
+from .dbkits import APIFinder
+from . import daclickhouse
+from . import damysql
+
+__all__ = [
+    'AbstractConnector',
+    'MySQLAPIConnector',
+    'MySQLDirectAccess',
+    'ClickHouseAPIConnector',
+    'ClickHouseDirectAccess',
+    'SQLServerAPIConnector',
+    'OracleAPIConnector',
+    'GaussAPIConnector',
+    'KingBaseAPIConnector',
+    'DaMengAPIConnector',
+    'PostgreSQLAPIConnector',
+    'DeepEngineAPIConnector',
+    'DeepModelAPIConnector',
+    'DeepModelKingBaseAPIConnector',
+]
+
+
+class AbstractConnector(ABC):  # pragma: no cover
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    async def execute(self, sql: str):
+        pass
+
+    @abstractmethod
+    async def execute_many(self, sqls: List[str]):
+        pass
+
+    @abstractmethod
+    async def trxn_execute(self, sqls: List[str]):
+        pass
+
+    @abstractmethod
+    async def select(self, sql: str):
+        pass
+
+    @abstractmethod
+    async def query_dataframe(self, sql: str) -> pd.DataFrame:
+        pass
+
+
+class MySQLAPIConnector(AbstractConnector):
+    def __init__(  # noqa
+        self,
+        version: Union[float, str] = None,
+    ):
+        self._v = version
+
+    @cache_async
+    async def build_api(self) -> MySQLAPI:
+        return await APIFinder().find_api(MySQLAPI, version=self._v)
+
+    async def execute(self, sql: str):
+        api = await self.build_api()
+        return await api.dml.run_sql(sql)
+
+    async def execute_many(self, sqls: List[str]):
+        api = await self.build_api()
+        return await asyncio.gather(
+            *(api.dml.run_sql(sql) for sql in sqls)
+        )
+
+    async def trxn_execute(self, sqls: List[str]):
+        api = await self.build_api()
+        return await api.dml.execute_batch_sql(sqls)
+
+    select = execute
+
+    async def query_dataframe(self, sql: str) -> pd.DataFrame:
+        api = await self.build_api()
+        resp = await api.dml.run_sql(sql)
+        select_data = [item or {} for item in resp.selectResult]
+        return pd.DataFrame.from_records(select_data)
+
+
+class ClickHouseAPIConnector(MySQLAPIConnector):
+    async def trxn_execute(self, sqls: List[str]):
+        raise NotImplementedError('ClickHouse does not support transaction')
+
+    @cache_async
+    async def build_api(self) -> ClickHouseAPI:
+        return await APIFinder().find_api(ClickHouseAPI, version=self._v)
+
+
+class MySQLDirectAccess(AbstractConnector):
+    async def execute(self, sql: str):
+        return await damysql.execute(sql)
+
+    async def execute_many(self, sqls: List[str]):
+        return await asyncio.gather(
+            *(damysql.execute(sql) for sql in sqls)
+        )
+
+    async def trxn_execute(self, sqls: List[str]):
+        return await damysql.trxn_execute(sqls)
+
+    async def select(self, sql: str):
+        return await damysql.select(sql)
+
+    async def query_dataframe(self, sql: str) -> pd.DataFrame:
+        return await damysql.query_dataframe(sql)
+
+
+class ClickHouseDirectAccess(AbstractConnector):
+    async def execute(self, sql: str):
+        return await daclickhouse.execute(sql)
+
+    async def execute_many(self, sqls: List[str]):
+        return await asyncio.gather(
+            *(daclickhouse.execute(sql) for sql in sqls)
+        )
+
+    async def trxn_execute(self, sqls: List[str]):
+        raise NotImplementedError('ClickHouse does not support transaction')
+
+    async def select(self, sql: str):
+        return await daclickhouse.select(sql)
+
+    async def query_dataframe(self, sql: str) -> pd.DataFrame:
+        return await daclickhouse.query_dataframe(sql)
+
+    async def insert_dataframe(self, table: str, dataframe: pd.DataFrame):  # noqa
+        return await daclickhouse.insert_dataframe(table, dataframe)
+
+
+class SQLServerAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> SQLServerAPI:
+        return await APIFinder().find_api(SQLServerAPI, version=self._v)
+
+
+class OracleAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> OracleAPI:
+        return await APIFinder().find_api(OracleAPI, version=self._v)
+
+
+class KingBaseAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> KingBaseAPI:
+        return await APIFinder().find_api(KingBaseAPI, version=self._v)
+
+
+class GaussAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> GaussAPI:
+        return await APIFinder().find_api(GaussAPI, version=self._v)
+
+
+class DaMengAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> DaMengAPI:
+        return await APIFinder().find_api(DaMengAPI, version=self._v)
+
+
+class PostgreSQLAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> PostgreSQLAPI:
+        return await APIFinder().find_api(PostgreSQLAPI, version=self._v)
+
+
+class DeepEngineAPIConnector(ClickHouseAPIConnector):
+    @cache_async
+    async def build_api(self) -> DeepEngineAPI:
+        return await APIFinder().find_api(DeepEngineAPI, version=self._v)
+
+    async def trxn_execute(self, sqls: List[str]):
+        raise NotImplementedError('DeepEngine does not support transaction')
+
+
+class DeepModelAPIConnector(MySQLAPIConnector):
+    @cache_async
+    async def build_api(self) -> DeepModelAPI:
+        return await APIFinder().find_api(DeepModelAPI, version=self._v)
+
+
+class DeepModelKingBaseAPIConnector(KingBaseAPIConnector):
+    @cache_async
+    async def build_api(self) -> DeepModelKingBaseAPI:
+        return await APIFinder().find_api(DeepModelKingBaseAPI, version=self._v)

deepfos/db/daclickhouse.py

@@ -0,0 +1,171 @@
+import time
+from reprlib import aRepr
+import warnings
+
+from loguru import logger
+from cachetools import TTLCache
+import clickhouse_driver
+import threading
+
+from deepfos import OPTION
+from deepfos.api.datatable import ClickHouseAPI
+from deepfos.lib.constant import UNSET
+from deepfos.lib.asynchronous import register_on_loop_shutdown
+from .utils import (
+    decrypt, AbsLeaseManager, ACCOUNT_EXPIRE,
+    PENDING, INITIALIZED, INITIALIZING, DBConnecetionError
+)
+
+aRepr.maxstring = 200
+
+
+# cdef class ClientCH:
+#     cdef:
+#         object  conn
+#         int status
+#         object  inited
+#         object  lease
+#         bytes secret
+
+class ClientCH:
+    def __init__(self):
+        self.conn = UNSET
+        self.status = PENDING
+        self.inited = threading.Event()
+        self.lease = LeaseManagerCH(ACCOUNT_EXPIRE / 3)
+        self.secret = "!ABCD-EFGH-IJKL@".encode()
+
+    async def establish_conn(self):
+        if self.status == INITIALIZED:
+            return self.conn
+
+        if self.status == PENDING:
+            self.status = INITIALIZING
+        else:
+            self.inited.wait()
+            if self.status != INITIALIZED:
+                raise RuntimeError("Failed to initialze connection pool.")
+            return self.conn
+
+        conf = await self.lease.renew()
+        self.lease.schedule(slow_start=True)
+        register_on_loop_shutdown(self.close)
+
+        conn_info = {
+            'host': conf.host,
+            'database': conf.dbName,
+            'user': conf.name,
+            'password': decrypt(self.secret, conf.password),
+            'port': conf.port,
+            'settings': {'use_numpy': True}
+        }
+
+        try:
+            self.conn = clickhouse_driver.Client(**conn_info)
+            self.conn.connection.force_connect()
+        except Exception as e:
+            self.status = PENDING
+            raise DBConnecetionError(e) from None
+        if not self.conn.connection.connected:
+            self.status = PENDING
+            raise DBConnecetionError("Not a valid connection")
+
+        self.status = INITIALIZED
+        self.inited.set()
+        return self.conn
+
+    async def ensure_connected(self):
+        max_retry = 3
+        retries = 0
+        interval = 0.5
+
+        while self.conn is UNSET or not self.conn.connection.connected:
+            try:
+                await self.establish_conn()
+            except DBConnecetionError:
+                retries += 1
+                if retries > max_retry:
+                    self.inited.set()
+                    raise
+                logger.exception(f'Failed to establish connection, '
+                                 f'starting {retries} times retry.')
+                time.sleep(interval)
+                interval *= 2
+            except Exception:
+                self.inited.set()
+                self.status = PENDING
+                raise
+
+        return self.conn
+
+    async def execute(self, sql):
+        conn = await self.ensure_connected()
+        logger.opt(lazy=True).debug("Run sql: {sql}", sql=lambda: aRepr.repr(sql))
+        return conn.execute(sql)
+
+    async def select(self, sql):
+        conn = await self.ensure_connected()
+        logger.opt(lazy=True).debug("Run sql: {sql}", sql=lambda: aRepr.repr(sql))
+        return conn.execute(sql)
+
+    async def query_dataframe(self, sql):
+        conn = await self.ensure_connected()
+        logger.opt(lazy=True).debug("Run sql: {sql}", sql=lambda: aRepr.repr(sql))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            return conn.query_dataframe(sql)
+
+    async def insert_dataframe(self, table, dataframe):
+        query = f"INSERT INTO `{table}` VALUES"
+        conn = await self.ensure_connected()
+        logger.debug(f"Run sql : {query} ...")
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            return conn.insert_dataframe(query, dataframe)
+
+    def close(self):
+        self.lease.cancel()
+        self.status = PENDING
+
+        if self.conn is not UNSET:
+            self.conn.disconnect()
+            self.conn = UNSET
+
+
+# cdef class ClientManager:
+class ClientManager:
+    cache = TTLCache(maxsize=10, ttl=1800)
+
+    def new_client(self):
+        key = (
+            OPTION.api.header.get('space'),
+            OPTION.api.header.get('app'),
+        )
+
+        if key not in self.cache:
+            self.cache[key] = ClientCH()
+
+        return self.cache[key]
+
+
+class LeaseManagerCH(AbsLeaseManager):
+    DB = 'clickhouse'
+
+    async def new_api(self):
+        return await ClickHouseAPI(version=1.0, sync=False)
+
+
+async def select(sql):
+    return await ClientManager().new_client().select(sql)
+
+
+async def execute(sql):
+    return await ClientManager().new_client().execute(sql)
+
+
+async def query_dataframe(sql):
+    return await ClientManager().new_client().query_dataframe(sql)
+
+
+async def insert_dataframe(table, dataframe):
+    return await ClientManager().new_client().insert_dataframe(table, dataframe)

deepfos/db/dameng.py

@@ -0,0 +1,101 @@
+"""Dameng客户端"""
+import asyncio
+from typing import Union, List, Iterable, TYPE_CHECKING
+
+import pandas as pd
+
+from deepfos.api.datatable import DaMengAPI
+from deepfos.cache import Manager, SpaceSeperatedTTLCache
+from deepfos.lib.utils import cachedclass
+from deepfos.lib.decorator import singleton, cached_property
+from deepfos.db.oracle import OracleDFSQLConvertor
+from .dbkits import SyncMeta, T_DataInfo, BaseSqlParser
+from .connector import DaMengAPIConnector
+from .mysql import _AbsAsyncMySQLClient  # noqa
+
+
+__all__ = [
+    'DaMengClient',
+    'AsyncDaMengClient',
+]
+
+
+@singleton
+class SqlParser(BaseSqlParser):
+    api_cls = DaMengAPI
+
+    @cached_property
+    def datatable_cls(self):
+        from deepfos.element.datatable import AsyncDataTableDaMeng
+        return AsyncDataTableDaMeng
+
+    @staticmethod
+    async def query_table_names(api: DaMengAPI, query_table):  # pragma: no cover
+        async def query_single(tbl_ele):
+            return tbl_ele.elementName, await api.dml.get_tablename(tbl_ele)
+
+        tablenames = await asyncio.gather(*(
+            query_single(table)
+            for table in query_table
+        ))
+
+        if len(tablenames) != len(query_table):
+            missing = set(t.elementName for t in query_table).difference(
+                set(t[0] for t in tablenames))
+
+            raise ValueError(f"Cannot resolve actual table names for element: {missing}")
+        return dict(tablenames)
+
+
+# -----------------------------------------------------------------------------
+# core
+class _AsyncDaMengClient(_AbsAsyncMySQLClient):
+    convertor = OracleDFSQLConvertor(quote_char='"')
+
+    def __init__(self, version: Union[float, str] = None):  # noqa
+        self.parser = SqlParser()
+        self.connector = DaMengAPIConnector(version)
+
+
+class _DaMengClient(_AsyncDaMengClient, metaclass=SyncMeta):
+    synchronize = (
+        'exec_sqls',
+        'query_dfs',
+        'insert_df',
+    )
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def exec_sqls(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ):
+            ...
+
+        def query_dfs(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+            ...
+
+        def insert_df(
+            self,
+            dataframe: pd.DataFrame,
+            element_name: str = None,
+            table_name: str = None,
+            updatecol: Iterable[str] = None,
+            table_info: T_DataInfo = None,
+            chunksize: int = None,
+        ):
+            ...
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class AsyncDaMengClient(_AsyncDaMengClient):
+    pass
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class DaMengClient(_DaMengClient):
+    pass