deepfos 1.1.60__py3-none-any.whl

deepfos/db/mysql.py

@@ -0,0 +1,184 @@
+"""数据库连接类"""
+import asyncio
+
+from cachetools import TTLCache
+
+from deepfos.lib.decorator import cached_property
+from typing import Union, List, Iterable, TYPE_CHECKING
+
+import pandas as pd
+
+from deepfos.api.datatable import MySQLAPI
+from deepfos.lib.utils import cachedclass
+from deepfos.lib.decorator import singleton
+from deepfos.cache import Manager
+from deepfos.options import OPTION
+from .dbkits import BaseSqlParser, SyncMeta, DataframeSQLConvertor, T_DataInfo, escape_mysql_string
+from .connector import MySQLAPIConnector, MySQLDirectAccess
+
+
+__all__ = [
+    'MySQLClient',
+    'AsyncMySQLClient',
+]
+
+
+@singleton
+class SqlParser(BaseSqlParser):
+    api_cls = MySQLAPI
+
+    @cached_property
+    def datatable_cls(self):
+        from deepfos.element.datatable import AsyncDataTableMySQL
+        return AsyncDataTableMySQL
+
+
+class MySQLConvertor(DataframeSQLConvertor):
+    escape_string = escape_mysql_string
+
+
+# -----------------------------------------------------------------------------
+# core
+class _AbsAsyncMySQLClient:
+    connector_cls = None
+    convertor = MySQLConvertor(quote_char='`')
+
+    def __init__(self, version: Union[float, str] = None):
+        self.parser = SqlParser()
+        self.connector = self.connector_cls(version)
+
+    async def exec_sqls(
+        self,
+        sqls: Union[str, Iterable[str]],
+        table_info: T_DataInfo = None
+    ):
+        """以事务执行多句sql
+
+        Args:
+            sqls: 要执行的sql语句，str或多句sql的list
+            table_info: sql中表名占位符对应的数据表元素信息。
+
+        """
+
+        if isinstance(sqls, str):
+            sqls = [sqls]
+
+        parsed_sql = await self.parser.parse(sqls, table_info)
+        resp = await self.connector.trxn_execute(parsed_sql)
+        return resp
+
+    async def query_dfs(
+        self,
+        sqls: Union[str, Iterable[str]],
+        table_info: T_DataInfo = None
+    ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+        """执行sql查询语句
+
+        获取DataFrame格式的二维表
+
+        Args:
+            sqls: 执行的sql，表名可以通过${table_name}占位
+            table_info: sql中表名占位符对应的数据表元素信息。
+
+        Notes:
+            如果执行的sql中没有任何表名占位符，sql将直接执行。
+            如果有占位符, 例如 ``${table1}``，那么要求 ``table_info`` 有
+            key值为 ``table1`` , 对应键值为包含
+            ``elementName, elementType, folderId/path`` 的字典，
+            或者 :class:`DataTableMySQL` 类型，或者 :class:`ElementModel` 类型
+
+        Returns:
+            :class:`DataFrame` 格式的二维数据表
+
+        """
+        if isinstance(sqls, str):
+            sql_list = await self.parser.parse([sqls], table_info)
+            return await self.connector.query_dataframe(sql_list[0])
+        else:
+            sqls = await self.parser.parse(sqls, table_info)
+            dfs = await asyncio.gather(
+                *(self.connector.query_dataframe(sql) for sql in sqls)
+            )
+            return list(dfs)
+
+    async def insert_df(
+        self,
+        dataframe: pd.DataFrame,
+        element_name: str = None,
+        table_name: str = None,
+        updatecol: Iterable[str] = None,
+        table_info: T_DataInfo = None,
+        chunksize: int = None,
+    ):
+        """将 :class:`DataFrame` 的插入数据表
+
+        Args:
+            dataframe: 入库数据
+            element_name: 数据表元素名
+            table_name: 数据表的 **实际表名**
+            updatecol: 更新的列 (用于INSERT INTO ON DUPLICATE)
+            table_info: 数据表元素信息，使用table
+            chunksize: 单次插库的数据行数
+
+        """
+        if table_name is not None:
+            tbl_name = table_name
+        elif element_name is not None:
+            tbl_name = (await self.parser.parse(["${%s}" % element_name], table_info))[0]
+        else:
+            raise ValueError("Either 'element_name' or 'table_name' must be presented.")
+
+        sqls = list(self.convertor.iter_sql(
+            dataframe, tbl_name, updatecol=updatecol, chunksize=chunksize
+        ))
+        return await self.connector.trxn_execute(sqls)
+
+
+class _AbsMySQLClient(_AbsAsyncMySQLClient, metaclass=SyncMeta):
+    synchronize = (
+        'exec_sqls',
+        'query_dfs',
+        'insert_df',
+    )
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def exec_sqls(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ):
+            ...
+
+        def query_dfs(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+            ...
+
+        def insert_df(
+            self,
+            dataframe: pd.DataFrame,
+            element_name: str = None,
+            table_name: str = None,
+            updatecol: Iterable[str] = None,
+            table_info: T_DataInfo = None,
+            chunksize: int = None,
+        ):
+            ...
+
+
+if OPTION.general.db_direct_access:
+    CONN_CLS = MySQLDirectAccess
+else:
+    CONN_CLS = MySQLAPIConnector
+
+
+@cachedclass(Manager.create_cache(TTLCache, maxsize=5, ttl=3600))
+class AsyncMySQLClient(_AbsAsyncMySQLClient):
+    connector_cls = CONN_CLS
+
+
+@cachedclass(Manager.create_cache(TTLCache, maxsize=5, ttl=3600))
+class MySQLClient(_AbsMySQLClient):
+    connector_cls = CONN_CLS

deepfos/db/oracle.py

@@ -0,0 +1,131 @@
+"""Oracle客户端"""
+import asyncio
+from deepfos.lib.decorator import cached_property
+from typing import Union, List, Iterable, TYPE_CHECKING
+
+import pandas as pd
+
+from deepfos.api.datatable import OracleAPI
+from deepfos.cache import Manager, SpaceSeperatedTTLCache
+from deepfos.lib.utils import cachedclass
+from deepfos.lib.decorator import singleton
+from .dbkits import BaseSqlParser, SyncMeta, T_DataInfo, DataframeSQLConvertor
+from .connector import OracleAPIConnector
+from .mysql import _AbsAsyncMySQLClient  # noqa
+
+
+__all__ = [
+    'OracleClient',
+    'AsyncOracleClient',
+    'OracleDFSQLConvertor'
+]
+
+
+class OracleDFSQLConvertor(DataframeSQLConvertor):
+    def build_sql(
+        self,
+        columns: str,
+        values_in_line: Iterable[str],
+        tablename: str,
+        updatecol: Iterable[str] = None,
+        **opts
+    ):
+        if updatecol is not None:
+            raise NotImplementedError("`updatecol` is not yet implemented for OracleDB.")
+
+        inserts = '\n'.join(
+            f'INTO {self.quote_char}{tablename.upper()}{self.quote_char} ({columns}) VALUES {value}'
+            for value in values_in_line
+        )
+
+        return f'INSERT ALL {inserts} SELECT 1 FROM DUAL'
+
+    def build_column_string(self, columns):
+        return ','.join(columns.map(lambda x: f'"{x.upper()}"'))
+
+    @staticmethod
+    def format_datetime(maybe_datetime):
+        return f'TO_DATE(\'' + \
+               maybe_datetime.dt.strftime("%Y-%m-%d %H:%M:%S") + \
+               '\', \'YYYY-MM-DD HH24:MI:SS\')'
+
+
+@singleton
+class SqlParser(BaseSqlParser):
+    api_cls = OracleAPI
+
+    @cached_property
+    def datatable_cls(self):
+        from deepfos.element.datatable import AsyncDataTableOracle
+        return AsyncDataTableOracle
+
+    @staticmethod
+    async def query_table_names(api: OracleAPI, query_table):
+        async def query_single(tbl_ele):
+            return tbl_ele.elementName, await api.dml.get_tablename(tbl_ele)
+
+        tablenames = await asyncio.gather(*(
+            query_single(table)
+            for table in query_table
+        ))
+
+        if len(tablenames) != len(query_table):
+            missing = set(t.elementName for t in query_table).difference(
+                set(t[0] for t in tablenames))
+
+            raise ValueError(f"Cannot resolve actual table names for element: {missing}")
+        return dict(tablenames)
+
+
+# -----------------------------------------------------------------------------
+# core
+class _AsyncOracleClient(_AbsAsyncMySQLClient):
+    convertor = OracleDFSQLConvertor(quote_char='"')
+
+    def __init__(self, version: Union[float, str] = None):  # noqa
+        self.parser = SqlParser()
+        self.connector = OracleAPIConnector(version)
+
+
+class _OracleClient(_AsyncOracleClient, metaclass=SyncMeta):
+    synchronize = (
+        'exec_sqls',
+        'query_dfs',
+        'insert_df',
+    )
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def exec_sqls(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ):
+            ...
+
+        def query_dfs(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+            ...
+
+        def insert_df(
+            self,
+            dataframe: pd.DataFrame,
+            element_name: str = None,
+            table_name: str = None,
+            updatecol: Iterable[str] = None,
+            table_info: T_DataInfo = None,
+            chunksize: int = None,
+        ):
+            ...
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class AsyncOracleClient(_AsyncOracleClient):
+    pass
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class OracleClient(_OracleClient):
+    pass

deepfos/db/postgresql.py

@@ -0,0 +1,192 @@
+"""PostgreSQL客户端"""
+from functools import cached_property
+from typing import Union, List, Iterable, TYPE_CHECKING
+
+import pandas as pd
+
+from deepfos.api.datatable import PostgreSQLAPI
+from deepfos.cache import Manager, SpaceSeperatedTTLCache
+from deepfos.lib.utils import cachedclass
+from deepfos.lib.decorator import singleton
+from .dbkits import BaseSqlParser, SyncMeta, T_DataInfo, DataframeSQLConvertor, escape_pg_string
+from .connector import PostgreSQLAPIConnector
+from .mysql import _AbsAsyncMySQLClient  # noqa
+
+
+__all__ = [
+    'PostgreSQLClient',
+    'AsyncPostgreSQLClient',
+    '_AsyncPostgreSQLClient',
+    'PostgreSQLConvertor',
+]
+
+
+@singleton
+class SqlParser(BaseSqlParser):
+    api_cls = PostgreSQLAPI
+
+    @cached_property
+    def datatable_cls(self):
+        from deepfos.element.datatable import AsyncDataTablePostgreSQL
+        return AsyncDataTablePostgreSQL
+
+
+class PostgreSQLConvertor(DataframeSQLConvertor):
+    escape_string = escape_pg_string
+
+    def iter_sql(
+        self,
+        dataframe: pd.DataFrame,
+        tablename: str,
+        updatecol: Iterable = None,
+        chunksize: int = None,
+        conflict_target: Iterable[str] = None,
+        **opts
+    ) -> Iterable[str]:
+        """ :class:`DataFrame` 对象转换为sql生成器
+
+        如果传了updatecol，将使用 ``INSERT INTO ON CONFLICT`` 语法
+
+        Args:
+            dataframe: 待插入数据
+            tablename: 数据库表名
+            updatecol: 更新的列
+            chunksize: 单条sql对应的最大dataframe行数
+            conflict_target: 使用INSERT INTO ON CONFLICT语法时的conflict基准列信息
+
+        Returns:
+            sql语句生成器
+
+        See Also:
+            :func:`df_to_sql`
+
+        """
+        # 获取sql
+        return super().iter_sql(dataframe, tablename, updatecol, chunksize, conflict_target=conflict_target, **opts)
+
+    def build_sql(
+        self,
+        columns: str,
+        values_in_line: Iterable[str],
+        tablename: str,
+        updatecol: Iterable[str] = None,
+        conflict_target: Iterable[str] = None,
+        **opts
+    ):
+        values = ','.join(values_in_line)
+        if updatecol is None:
+            return f'INSERT INTO {self.quote_char}{tablename}{self.quote_char} ({columns}) VALUES {values}'
+
+        update_str = ','.join([
+            f"{self.quote_char}{x}{self.quote_char}="
+            f"EXCLUDED.{self.quote_char}{x}{self.quote_char}"
+            for x in updatecol
+        ])
+        if not update_str:
+            return f'INSERT INTO {self.quote_char}{tablename}{self.quote_char} ({columns}) VALUES {values}'
+
+        if conflict_target is None:
+            raise ValueError('如需使用ON CONFLICT DO UPDATE语法，'
+                             '需提供有唯一约束的列作为conflict_target列信息')
+
+        conflict_target_clause = ",".join([
+            f"{self.quote_char}{x}{self.quote_char}"
+            for x in conflict_target
+        ])
+
+        if conflict_target_clause:
+            conflict_target_clause = f"({conflict_target_clause})"
+
+        return f'INSERT INTO {self.quote_char}{tablename}{self.quote_char} ({columns}) ' \
+               f'VALUES {values} ' \
+               f'ON CONFLICT {conflict_target_clause} ' \
+               f'DO UPDATE SET {update_str}'
+
+
+# -----------------------------------------------------------------------------
+# core
+class _AsyncPostgreSQLClient(_AbsAsyncMySQLClient):
+    convertor = PostgreSQLConvertor(quote_char='"')
+
+    def __init__(self, version: Union[float, str] = None):  # noqa
+        self.parser = SqlParser()
+        self.connector = PostgreSQLAPIConnector(version)
+
+    async def insert_df(
+        self,
+        dataframe: pd.DataFrame,
+        element_name: str = None,
+        table_name: str = None,
+        updatecol: Iterable[str] = None,
+        table_info: T_DataInfo = None,
+        chunksize: int = None,
+        conflict_target: Iterable[str] = None,
+    ):
+        """将 :class:`DataFrame` 的插入数据表
+
+        Args:
+            dataframe: 入库数据
+            element_name: 数据表元素名
+            table_name: 数据表的 **实际表名**
+            updatecol: 更新的列 (用于INSERT INTO ON CONFLICT)
+            table_info: 数据表元素信息，使用table
+            chunksize: 单次插库的数据行数
+            conflict_target: 使用INSERT INTO ON CONFLICT语法时的约束列信息
+
+        """
+        if table_name is not None:
+            tbl_name = table_name
+        elif element_name is not None:
+            tbl_name = (await self.parser.parse(["${%s}" % element_name], table_info))[0]
+        else:
+            raise ValueError("Either 'element_name' or 'table_name' must be presented.")
+
+        sqls = list(self.convertor.iter_sql(
+            dataframe, tbl_name, updatecol=updatecol, chunksize=chunksize, conflict_target=conflict_target
+        ))
+        return await self.connector.trxn_execute(sqls)
+
+
+class _PostgreSQLClient(_AsyncPostgreSQLClient, metaclass=SyncMeta):
+    synchronize = (
+        'exec_sqls',
+        'query_dfs',
+        'insert_df',
+    )
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def exec_sqls(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ):
+            ...
+
+        def query_dfs(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+            ...
+
+        def insert_df(
+            self,
+            dataframe: pd.DataFrame,
+            element_name: str = None,
+            table_name: str = None,
+            updatecol: Iterable[str] = None,
+            table_info: T_DataInfo = None,
+            chunksize: int = None,
+            conflict_target: Iterable[str] = None,
+        ):
+            ...
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class AsyncPostgreSQLClient(_AsyncPostgreSQLClient):
+    pass
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class PostgreSQLClient(_PostgreSQLClient):
+    pass

deepfos/db/sqlserver.py

@@ -0,0 +1,99 @@
+"""SQLServer客户端"""
+from deepfos.lib.decorator import cached_property
+from typing import Union, List, Iterable, TYPE_CHECKING
+
+import pandas as pd
+
+from deepfos.api.datatable import SQLServerAPI
+from deepfos.cache import Manager, SpaceSeperatedTTLCache
+from deepfos.lib.utils import cachedclass
+from deepfos.lib.decorator import singleton
+from .dbkits import BaseSqlParser, SyncMeta, T_DataInfo
+from .connector import SQLServerAPIConnector
+from .mysql import _AbsAsyncMySQLClient, MySQLConvertor  # noqa
+
+
+__all__ = [
+    'SQLServerClient',
+    'AsyncSQLServerClient',
+    'SQLServerDFSQLConvertor',
+]
+
+
+@singleton
+class SqlParser(BaseSqlParser):
+    api_cls = SQLServerAPI
+
+    @cached_property
+    def datatable_cls(self):
+        from deepfos.element.datatable import AsyncDataTableSQLServer
+        return AsyncDataTableSQLServer
+
+
+class SQLServerDFSQLConvertor(MySQLConvertor):
+    def build_sql(
+        self,
+        columns: str,
+        values_in_line: Iterable[str],
+        tablename: str,
+        updatecol: Iterable[str] = None,
+        **opts
+    ):
+        if updatecol is not None:
+            raise NotImplementedError("`updatecol` is not yet implemented for SQLServerDB.")
+
+        return super().build_sql(columns, values_in_line, tablename, None, **opts)
+
+
+# -----------------------------------------------------------------------------
+# core
+class _AsyncSQLServerClient(_AbsAsyncMySQLClient):
+    convertor = SQLServerDFSQLConvertor(quote_char="")
+
+    def __init__(self, version: Union[float, str] = None):  # noqa
+        self.parser = SqlParser()
+        self.connector = SQLServerAPIConnector(version)
+
+
+class _SQLServerClient(_AsyncSQLServerClient, metaclass=SyncMeta):
+    synchronize = (
+        'exec_sqls',
+        'query_dfs',
+        'insert_df',
+    )
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def exec_sqls(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ):
+            ...
+
+        def query_dfs(
+            self,
+            sqls: Union[str, Iterable[str]],
+            table_info: T_DataInfo = None
+        ) -> Union[pd.DataFrame, List[pd.DataFrame]]:
+            ...
+
+        def insert_df(
+            self,
+            dataframe: pd.DataFrame,
+            element_name: str = None,
+            table_name: str = None,
+            updatecol: Iterable[str] = None,
+            table_info: T_DataInfo = None,
+            chunksize: int = None,
+        ):
+            ...
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class AsyncSQLServerClient(_AsyncSQLServerClient):
+    pass
+
+
+@cachedclass(Manager.create_cache(SpaceSeperatedTTLCache, maxsize=5, ttl=3600))
+class SQLServerClient(_SQLServerClient):
+    pass