quantnodes 3.0.0__py3-none-any.whl

QuantNodes/database_node/base.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+"""数据库节点基类
+
+定义所有数据库节点的统一接口
+"""
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+import pandas as pd
+
+from QuantNodes.core.data_source import DataSource
+
+
+class BaseDBNode(DataSource, ABC):
+    """数据库节点基类
+
+    所有数据库节点必须实现以下接口：
+
+    Methods:
+        connect(): 建立连接
+        query(sql, params): 执行查询，返回 DataFrame
+        execute(sql, params): 执行 DDL/DML，返回影响行数
+        insert_df(df, table, if_exists): 插入 DataFrame
+        disconnect(): 关闭连接
+        health_check(): 健康检查
+
+    Note:
+        继承 ``DataSource`` (Phase 3.3) 统一生命周期; ``close()`` 默认
+        委托 ``disconnect()``, 子类无需改动。
+    """
+
+    _conn: Any = None
+
+    @abstractmethod
+    def connect(self) -> Any:
+        """建立数据库连接"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def query(self, sql: str, params: Optional[tuple] = None) -> pd.DataFrame:
+        """执行 SQL 查询，返回 DataFrame
+
+        Args:
+            sql: SQL 查询语句
+            params: 查询参数（可选）
+
+        Returns:
+            pd.DataFrame 查询结果
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def execute(self, sql: str, params: Optional[tuple] = None) -> int:
+        """执行 DDL/DML 语句
+
+        Args:
+            sql: SQL 语句
+            params: 语句参数（可选）
+
+        Returns:
+            int 影响行数
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def insert_df(self, df: pd.DataFrame, table: str,
+                  if_exists: str = 'append') -> int:
+        """插入 DataFrame 到数据库
+
+        Args:
+            df: 要插入的 DataFrame
+            table: 目标表名
+            if_exists: 表存在时的行为 ('append', 'replace', 'fail')
+
+        Returns:
+            int 插入行数
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def disconnect(self) -> None:
+        """关闭数据库连接"""
+        raise NotImplementedError
+
+    def close(self) -> None:
+        """释放资源 (DataSource 协议)。默认委托 ``disconnect()``。"""
+        self.disconnect()
+
+    def health_check(self) -> bool:
+        """健康检查
+
+        Returns:
+            bool 连接是否正常
+        """
+        try:
+            self.query("SELECT 1")
+            return True
+        except Exception:
+            return False
+
+    def show_tables(self) -> list:
+        """列出所有表。默认通过 SHOW TABLES 实现。
+
+        Returns:
+            list[str] 表名列表。
+
+        Note:
+            ClickHouse 等需要在 schema 中限定表名的数据库可重写此方法
+            (例: ``SHOW TABLES FROM <database>``)。csv/parquet 等无 schema
+            概念的 backend 也应重写, 抛 NotImplementedError 或返回自定义列表。
+        """
+        result = self.query("SHOW TABLES")
+        return result.iloc[:, 0].tolist()
+
+    def show_databases(self) -> list:
+        """列出所有数据库/Schema。默认通过 SHOW DATABASES 实现。
+
+        Returns:
+            list[str] 数据库名列表。
+
+        Note:
+            单数据库 backend (sqlite, csv, parquet, 单 instance duckdb)
+            可重写返回 ``[self._database]`` 或抛 NotImplementedError。
+        """
+        result = self.query("SHOW DATABASES")
+        return result.iloc[:, 0].tolist()
+
+    def __enter__(self):
+        """上下文管理器入口"""
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """上下文管理器出口"""
+        self.disconnect()
+        return False

QuantNodes/database_node/clickhouse_node.py

@@ -0,0 +1,272 @@
+# -*- coding: utf-8 -*-
+"""ClickHouse 节点
+
+支持 HTTP 接口和官方 driver 双接口
+"""
+import gzip
+import http.client
+import json
+import urllib.parse
+from collections import namedtuple
+from typing import Optional
+
+import pandas as pd
+
+from QuantNodes.database_node.base import BaseDBNode
+
+
+ch_conn_tuple = namedtuple('ch_conn_tuple', ['host', 'port', 'user', 'passwd', 'db'])
+
+
+class CHBase:
+    """ClickHouse HTTP 基础实现"""
+
+    def __init__(self, name: str, user: str = 'default', passwd: str = '123456',
+                 host: str = '0.0.0.0', port: int = 8123, db: str = 'default'):
+        self.name = name
+        self._para = ch_conn_tuple(host, port, user, passwd, db)
+        self.accepted_formats = [
+            'DataFrame', 'TabSeparated', 'TabSeparatedRaw', 'TabSeparatedWithNames',
+            'TabSeparatedWithNamesAndTypes', 'CSV', 'CSVWithNames', 'Values', 'Vertical', 'JSON',
+            'JSONCompact', 'JSONEachRow', 'TSKV', 'Pretty', 'PrettyCompact',
+            'PrettyCompactMonoBlock', 'PrettyNoEscapes', 'PrettySpace', 'XML'
+        ]
+        self.settings = self._merge_settings(None)
+        http_get_params = {'user': self._para.user, 'password': self._para.passwd}
+        http_get_params.update(self.settings)
+        self.http_get_params = http_get_params
+
+    def SHOWTABLES(self):
+        """显示所有表"""
+        res = self.get(f'SHOW TABLES FROM {self._para.db}').values
+        return res
+
+    def _create_conn(self):
+        """创建 HTTP 连接"""
+        url_str = (f"http://{self._para.user}:{self._para.passwd}@{self._para.host}:{self._para.port}")
+        components = urllib.parse.urlparse(url_str)
+        return http.client.HTTPConnection(components.hostname, port=components.port)
+
+    @staticmethod
+    def _check_sql_select_only(sql: str) -> None:
+        """检查 SQL 是否为查询语句"""
+        if sql.strip(' \n\t').lower()[:6] not in ['select', 'descri', 'show t', 'show d']:
+            first_word = sql.strip(' \n\t').split(' ')[0]
+            raise ValueError(
+                f'"query" should start with "select" or "describe" or "show", but got "{first_word}"'
+            )
+
+    @staticmethod
+    def _transfer_sql_format(sql: str, convert_to: str) -> str:
+        """转换 SQL 格式"""
+        clickhouse_format = 'JSON' if convert_to is None else 'JSONCompact' if convert_to.lower() == 'dataframe' else convert_to
+        query_with_format = (sql.rstrip('; \n\t') + ' format ' + clickhouse_format).replace('\n', ' ').strip(' ')
+        return query_with_format
+
+    def _compression_switched_request(self, query_with_format: str, conn, updated_settings, http_get_params):
+        """根据设置决定是否压缩请求"""
+        if updated_settings['enable_http_compression'] == 1:
+            conn.request('POST', '/?' + urllib.parse.urlencode(http_get_params),
+                         body=gzip.compress(query_with_format.encode()),
+                         headers={'Content-Encoding': 'gzip', 'Accept-Encoding': 'gzip'})
+        else:
+            conn.request('POST', '/?' + urllib.parse.urlencode(http_get_params), body=query_with_format.encode())
+        return conn
+
+    def _get_data(self, conn, updated_settings, auto_close=True):
+        """获取响应数据"""
+        resp = conn.getresponse()
+
+        if resp.status == 404:
+            error_message = (gzip.decompress(resp.read()).decode() if updated_settings['enable_http_compression'] == 1
+                           else resp.read().decode())
+            if auto_close:
+                conn.close()
+            raise ValueError(error_message)
+        elif resp.status == 401:
+            if auto_close:
+                conn.close()
+            raise ConnectionRefusedError(resp.reason + '. The username or password is incorrect.')
+        else:
+            if resp.status != 200:
+                error_message = (gzip.decompress(resp.read()).decode() if updated_settings['enable_http_compression'] == 1
+                               else resp.read().decode())
+                if auto_close:
+                    conn.close()
+                raise NotImplementedError(f'Unknown Error: status: {resp.status}, reason: {resp.reason}, message: {error_message}')
+
+        total = bytes()
+        while not resp.isclosed():
+            total += resp.read(300 * 1024)
+        if auto_close:
+            conn.close()
+        return gzip.decompress(total).decode() if updated_settings['enable_http_compression'] == 1 else total.decode()
+
+    @staticmethod
+    def _load_into_pd(ret_value: str, convert_to: str) -> pd.DataFrame:
+        """将返回值转换为 DataFrame"""
+        if convert_to.lower() == 'dataframe':
+            result_dict = json.loads(ret_value, strict=False)
+            dataframe = pd.DataFrame.from_records(
+                result_dict['data'],
+                columns=[i['name'] for i in result_dict['meta']]
+            )
+            for i in result_dict['meta']:
+                if i['type'] in ['DateTime', 'Nullable(DateTime)']:
+                    dataframe[i['name']] = pd.to_datetime(dataframe[i['name']])
+            return dataframe
+        return ret_value
+
+    def get(self, sql: str, convert_to: str = 'DataFrame', auto_close: bool = True):
+        """执行查询并返回结果"""
+        conn = self._create_conn()
+        self._check_sql_select_only(sql)
+        updated_settings = self.settings
+        query_with_format = self._transfer_sql_format(sql, convert_to)
+        conn = self._compression_switched_request(query_with_format, conn, updated_settings, self.http_get_params)
+        ret_value = self._get_data(conn, updated_settings, auto_close=auto_close)
+        return self._load_into_pd(ret_value, convert_to)
+
+    @staticmethod
+    def _merge_settings(settings):
+        """合并设置"""
+        updated_settings = {
+            'enable_http_compression': 1,
+            'send_progress_in_http_headers': 0,
+            'log_queries': 1,
+            'connect_timeout': 10,
+            'receive_timeout': 300,
+            'send_timeout': 300,
+            'output_format_json_quote_64bit_integers': 0,
+            'wait_end_of_query': 0
+        }
+        if settings is not None:
+            invalid_keys = list(set(settings.keys()) - set(updated_settings.keys()))
+            if invalid_keys:
+                raise ValueError(f'setting "{invalid_keys[0]}" is invalid')
+            updated_settings.update(settings)
+
+        for i in updated_settings:
+            updated_settings[i] = 1 if updated_settings[i] is True else 0 if updated_settings[i] is False else updated_settings[i]
+        return updated_settings
+
+    def get_describe_table(self, db: str, table: str) -> pd.DataFrame:
+        """获取表结构"""
+        return self.get(f'DESCRIBE TABLE {db}.{table}', auto_close=True)
+
+    def query(self, sql: str) -> pd.DataFrame:
+        """执行查询"""
+        return self.get(sql, convert_to='DataFrame')
+
+
+class ClickHouseNode(BaseDBNode):
+    """ClickHouse 数据库节点
+
+    支持 HTTP 接口和官方 driver 双接口
+
+    Args:
+        host: 主机地址
+        port: 端口 (HTTP 默认 8123，Native 默认 9000)
+        user: 用户名 (默认 default)
+        passwd: 密码
+        database: 数据库名 (默认 default)
+        interface: 接口类型 ('http' 或 'native'，默认 'http')
+        pool_size: 连接池大小 (默认 10，可配置)
+
+    Example:
+        >>> # HTTP 接口
+        >>> node = ClickHouseNode(
+        ...     host="localhost",
+        ...     user="default",
+        ...     passwd="",
+        ...     database="default"
+        ... )
+
+        >>> # Native 接口
+        >>> node = ClickHouseNode(
+        ...     host="localhost",
+        ...     port=9000,
+        ...     interface="native"
+        ... )
+    """
+
+    def __init__(self, host: str, port: int = 8123,
+                 user: str = 'default', passwd: str = '',
+                 database: str = 'default',
+                 interface: str = 'http',
+                 pool_size: int = 10):
+        self._host = host
+        self._port = port
+        self._user = user
+        self._passwd = passwd
+        self._database = database
+        self._interface = interface
+        self._pool_size = pool_size
+        self._client = None
+        self._http_client = None
+
+    def connect(self):
+        """建立连接"""
+        if self._interface == 'native':
+            import clickhouse_connect
+            self._client = clickhouse_connect.get_client(
+                host=self._host,
+                port=self._port,
+                username=self._user,
+                password=self._passwd,
+                database=self._database,
+            )
+        else:
+            self._http_client = CHBase(
+                name=self._database,
+                user=self._user,
+                passwd=self._passwd,
+                host=self._host,
+                port=self._port,
+                db=self._database,
+            )
+        return self._client or self._http_client
+
+    def query(self, sql: str, params: Optional[tuple] = None) -> pd.DataFrame:
+        """执行查询"""
+        client = self._client or self._http_client or self.connect()
+        if self._interface == 'native':
+            result = client.query(sql)
+            if hasattr(result, 'result_rows'):
+                return pd.DataFrame(result.result_rows, columns=result.column_names)
+            return result
+        else:
+            return client.get(sql, convert_to='DataFrame')
+
+    def execute(self, sql: str, params: Optional[tuple] = None) -> int:
+        """执行 DDL/DML"""
+        client = self._client or self._http_client or self.connect()
+        if self._interface == 'native':
+            client.command(sql)
+            return 0
+        else:
+            client.query(sql)
+            return 0
+
+    def insert_df(self, df: pd.DataFrame, table: str,
+                  if_exists: str = 'append') -> int:
+        """插入 DataFrame"""
+        client = self._client or self._http_client or self.connect()
+        if self._interface == 'native':
+            client.insert_df(table, df)
+        else:
+            client.insert(df, self._database, table)
+        return len(df)
+
+    def disconnect(self) -> None:
+        """关闭连接"""
+        if self._client:
+            if self._interface == 'native':
+                self._client.close()
+            self._client = None
+        self._http_client = None
+
+    def show_tables(self) -> list:
+        """ClickHouse 需要按数据库限定表名 (SHOW TABLES FROM <db>)。"""
+        result = self.query(f"SHOW TABLES FROM {self._database}")
+        return result.iloc[:, 0].tolist()

QuantNodes/database_node/csv_node.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""CSV 读取节点
+
+支持 WHERE 子句过滤
+"""
+import os
+from typing import Optional
+
+import pandas as pd
+
+from QuantNodes.database_node.base import BaseDBNode
+
+
+class CSVNode(BaseDBNode):
+    """CSV 文件读取节点
+
+    支持 WHERE 子句过滤
+
+    Args:
+        filepath: CSV 文件绝对路径
+        encoding: 字符编码 (默认 utf-8)
+        sep: 分隔符 (默认 ,)
+
+    Example:
+        >>> node = CSVNode("/data/users.csv")
+        >>> # 全量读取
+        >>> df = node.query()
+        >>> # 带 WHERE 过滤
+        >>> df = node.query("SELECT * WHERE age > 18")
+    """
+
+    def __init__(self, filepath: str, encoding: str = 'utf-8', sep: str = ','):
+        self._filepath = filepath
+        self._encoding = encoding
+        self._sep = sep
+        self._data: Optional[pd.DataFrame] = None
+
+    def connect(self) -> pd.DataFrame:
+        """读取 CSV 到内存"""
+        self._data = pd.read_csv(
+            self._filepath,
+            encoding=self._encoding,
+            sep=self._sep
+        )
+        return self._data
+
+    def query(self, sql: Optional[str] = None, params: Optional[tuple] = None) -> pd.DataFrame:
+        """执行查询
+
+        Args:
+            sql: SQL 查询语句（可选），支持 WHERE 子句
+            params: 查询参数（暂未使用）
+
+        Returns:
+            pd.DataFrame 查询结果
+        """
+        data = self._data or self.connect()
+
+        if sql is None:
+            return data
+
+        if 'WHERE' in sql.upper():
+            where_clause = sql.split('WHERE', 1)[1].strip()
+            return data.query(where_clause)
+
+        return data
+
+    def execute(self, sql: str, params: Optional[tuple] = None) -> int:
+        """CSV 节点不支持 execute"""
+        raise NotImplementedError("CSVNode 不支持 execute 操作")
+
+    def insert_df(self, df: pd.DataFrame, table: str,
+                  if_exists: str = 'append') -> int:
+        """CSV 节点不支持 insert"""
+        raise NotImplementedError("CSVNode 不支持 insert 操作")
+
+    def disconnect(self) -> None:
+        """释放内存"""
+        self._data = None
+
+    def health_check(self) -> bool:
+        """健康检查"""
+        return os.path.exists(self._filepath)

QuantNodes/database_node/duckdb_node.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+"""DuckDB 节点
+
+支持内存模式和文件模式，支持只读模式
+"""
+from typing import Optional
+
+import pandas as pd
+
+from QuantNodes.database_node.base import BaseDBNode
+
+
+class DuckDBNode(BaseDBNode):
+    """DuckDB 数据库节点
+
+    支持内存模式和文件模式，支持只读模式
+
+    Args:
+        database: 数据库路径，`:memory:` 表示内存模式，
+                 绝对路径表示文件模式
+        read_only: 是否只读模式（仅文件模式有效，默认 False）
+
+    Example:
+        >>> # 内存模式
+        >>> node = DuckDBNode(":memory:")
+
+        >>> # 文件模式
+        >>> node = DuckDBNode("/data/analysis.duckdb")
+
+        >>> # 只读模式
+        >>> node = DuckDBNode("/data/analysis.duckdb", read_only=True)
+    """
+
+    def __init__(self, database: str = ":memory:", read_only: bool = False):
+        self._database = database
+        self._read_only = read_only
+        self._conn = None
+
+    def connect(self):
+        """建立 DuckDB 连接"""
+        import duckdb
+        self._conn = duckdb.connect(self._database, read_only=self._read_only)
+        return self._conn
+
+    def query(self, sql: str, params: Optional[tuple] = None) -> pd.DataFrame:
+        """执行查询"""
+        conn = self._conn or self.connect()
+        if params:
+            return conn.execute(sql, params).fetchdf()
+        return conn.execute(sql).fetchdf()
+
+    def execute(self, sql: str, params: Optional[tuple] = None) -> int:
+        """执行 DDL/DML"""
+        conn = self._conn or self.connect()
+        if params:
+            result = conn.execute(sql, params)
+        else:
+            result = conn.execute(sql)
+        try:
+            return result.rowcount
+        except Exception:
+            return 0
+
+    def insert_df(self, df: pd.DataFrame, table: str,
+                  if_exists: str = 'append') -> int:
+        """插入 DataFrame"""
+        conn = self._conn or self.connect()
+        conn.register('temp_df', df)
+        if if_exists == 'replace':
+            conn.execute(f"DROP TABLE IF EXISTS {table}")
+            conn.execute(f"CREATE TABLE {table} AS SELECT * FROM temp_df")
+        elif if_exists == 'append':
+            try:
+                conn.execute(f"INSERT INTO {table} SELECT * FROM temp_df")
+            except Exception:
+                conn.execute(f"CREATE TABLE {table} AS SELECT * FROM temp_df")
+        else:
+            conn.execute(f"INSERT INTO {table} SELECT * FROM temp_df")
+        conn.unregister('temp_df')
+        return len(df)
+
+    def disconnect(self) -> None:
+        """关闭连接"""
+        if self._conn:
+            self._conn.close()
+            self._conn = None

QuantNodes/database_node/factory.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""database_node 工厂 (Phase 3.3)
+
+将"按 source 字符串选择数据库后端"的逻辑收敛到单一注册表驱动工厂,
+取代散落在调用方 (如 ``agent/tools/config_backtest.py``) 的 if/elif 阶梯。
+
+Usage:
+    from QuantNodes.database_node import create_db_node
+
+    node = create_db_node("sqlite", database="/data/x.db")
+    node = create_db_node("clickhouse", host="localhost", database="default")
+
+    # 扩展新后端
+    from QuantNodes.database_node import register_db_node
+    register_db_node("myback", lambda **p: MyBackendNode(**p))
+
+注意: 工厂只负责 "参数 -> 实例"; 连接参数的来源 (conn.ini / path 解析)
+仍由调用方负责。
+"""
+from typing import Callable, Dict
+
+from QuantNodes.database_node.base import BaseDBNode
+from QuantNodes.database_node.sqlite_node import SQLiteNode
+from QuantNodes.database_node.duckdb_node import DuckDBNode
+from QuantNodes.database_node.mysql_node import MySQLNode
+from QuantNodes.database_node.clickhouse_node import ClickHouseNode
+from QuantNodes.database_node.csv_node import CSVNode
+from QuantNodes.database_node.parquet_node import ParquetNode
+
+DBNodeBuilder = Callable[..., BaseDBNode]
+
+_DB_NODE_BUILDERS: Dict[str, DBNodeBuilder] = {
+    "sqlite": lambda **p: SQLiteNode(**p),
+    "duckdb": lambda **p: DuckDBNode(**p),
+    "mysql": lambda **p: MySQLNode(**p),
+    "clickhouse": lambda **p: ClickHouseNode(**p),
+    "csv": lambda **p: CSVNode(**p),
+    "parquet": lambda **p: ParquetNode(**p),
+}
+
+
+def create_db_node(source: str, **params) -> BaseDBNode:
+    """按 source 字符串创建对应的数据库节点实例。
+
+    Args:
+        source: 后端类型, 见 ``available_sources()``。
+        **params: 透传给对应 Node 构造函数的关键字参数。
+
+    Returns:
+        BaseDBNode 子类实例。
+
+    Raises:
+        ValueError: source 未注册。
+    """
+    builder = _DB_NODE_BUILDERS.get(source)
+    if builder is None:
+        raise ValueError(
+            f"Unsupported data source: {source}. "
+            f"Available: {sorted(_DB_NODE_BUILDERS)}"
+        )
+    return builder(**params)
+
+
+def register_db_node(source: str, builder: DBNodeBuilder) -> None:
+    """注册新的后端 builder (供扩展)。
+
+    Args:
+        source: 后端类型字符串。
+        builder: 接收 **params 返回 BaseDBNode 实例的可调用对象。
+
+    Raises:
+        ValueError: source 为空或已存在。
+    """
+    if not source:
+        raise ValueError("source must be a non-empty string")
+    if source in _DB_NODE_BUILDERS:
+        raise ValueError(f"source '{source}' already registered")
+    _DB_NODE_BUILDERS[source] = builder
+
+
+def available_sources() -> list:
+    """返回已注册的后端类型列表 (排序)。"""
+    return sorted(_DB_NODE_BUILDERS)