dataquant 1.1.6__py3-none-any.whl

dataquant/sql/__init__.py

@@ -0,0 +1 @@
+# -*- coding: UTF-8 -*-

dataquant/sql/api.py

@@ -0,0 +1,61 @@
+# -*- coding: UTF-8 -*-
+import warnings
+
+import pandas as pd
+
+from dataquant.apis.base import get_data
+
+
+__all__ = [
+    "get_sql_data"
+]
+
+
+def get_sql_data(qry_db, qry_sql, page_no=-1, page_size=-1):
+    """
+    获取SQL查询数据
+
+    """
+
+    if qry_db is None or qry_sql is None:
+        warnings.warn("函数[get_sql_data]的参数(qry_db, qry_sql)为必填项")
+        return None
+
+    try:
+        if page_no == -1:
+            page_no = 0
+        if page_size == -1:
+            page_size = 100000
+
+        params = {
+            "query_db": qry_db,
+            "query_sql": qry_sql,
+            "page_no": page_no,
+            "page_size": page_size,
+            "api_type": "sql",
+        }
+
+        _url = "sql/get_sql_data" + "/" + qry_db
+        result = get_data(_url, **params)
+        if result.empty:
+            return result
+
+        total_cnt = result['query_total'][0]
+        while total_cnt > result.shape[0]:
+            params['page_no'] += 1
+            page_result = get_data(_url, **params)
+            result = pd.concat([result, page_result], axis=0)
+
+        result.reset_index(drop=True, inplace=True)
+        result.drop(['query_total'], axis=1, inplace=True)
+        columns = result.columns.tolist()
+        new_columns = [x.lower() for x in columns]
+        result.columns = new_columns
+
+        return result
+    except Exception as ex:
+        import traceback
+        print(traceback.print_exc())
+        return None
+
+

dataquant/utils/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: UTF-8 -*-
+
+from dataquant.utils.client import init
+
+init()

dataquant/utils/client.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+
+import time
+import random
+import warnings
+
+_CLIENT = None
+
+
+def get_client():
+    global _CLIENT
+    if _CLIENT is None:
+        init()
+
+    return _CLIENT
+
+
+def init(**kwargs):
+    """
+    :param str username: 用户名，license模式下为'license'
+    :param str password: 用户密码，license模式下为下发的license
+    :param str protocol：传输协议，默认HTTP
+    :param str url: 数据服务的URL
+    :param int connect_timeout: 连接建立超时时间,默认5秒
+    :param int request_timeout: 数据传输超时时间，默认300秒
+    :param str compressor: 数据传输过程中的压缩算法，默认不使用
+    :param int pool_size: 连接池大小，默认10
+    :param int page_size: 数据请求分页大小，默认100000
+    :return:
+    """
+
+    from dataquant.utils.config import read_config, write_config
+
+    # 获取系统配置
+    kwargs_num = len(kwargs)
+    retry_times = 0
+    load_succ = False
+    while not load_succ and retry_times <= 3:
+        try:
+            conf = read_config()['system']
+            load_succ = True
+        except Exception as ex:
+            time.sleep(random.random()*0.5+0.2)
+            retry_times = retry_times + 1
+    if not load_succ and retry_times > 3:
+        # 读取配置失败，使用默认配置启动
+        warnings.warn("读取配置失败，请检查dataquant.utils.config.yml")
+
+    user = kwargs.pop("username", None)
+    if user is None:
+        user = conf.get("username")
+    else:
+        conf["username"] = user
+    assert user
+
+    pwd = kwargs.pop("password", None)
+    if pwd is None:
+        pwd = conf.get("password")
+    else:
+        conf["password"] = pwd
+    assert pwd
+
+    url = kwargs.pop("url", None)
+    if url is None:
+        url = conf.get("url")
+    else:
+        conf["url"] = url
+    assert url
+
+    protocol = kwargs.pop("protocol", None)
+    if protocol is None:
+        protocol = conf.get("protocol", "HTTP")
+    else:
+        if protocol != "HTTP":
+            warnings.warn("protocol 当前只支持HTTP，设置的{}将不会生效".format(protocol))
+        else:
+            conf["protocol"] = "HTTP"
+
+    con_timeout = kwargs.pop("connect_timeout", None)
+    if con_timeout is None:
+        con_timeout = conf.get("connect_timeout", 5)
+    else:
+        if int(con_timeout) <= 0 or int(con_timeout) > 1000:
+            warnings.warn("connect_timeout 有效范围为(0, 1000]，设置的{}将不会生效".format(con_timeout))
+        else:
+            conf["connect_timeout"] = con_timeout
+
+    timeout = kwargs.pop("request_timeout", None)
+    if timeout is None:
+        timeout = conf.get("request_timeout", 300)
+    else:
+        if int(timeout) <= 0 or int(timeout) > 100000:
+            warnings.warn("request_timeout 有效范围为(0, 100000]，设置的{}将不会生效".format(timeout))
+        else:
+            conf["request_timeout"] = timeout
+
+    pool_size = kwargs.pop("pool_size", None)
+    if pool_size is None:
+        pool_size = conf.get("pool_size", 10)
+    else:
+        if int(pool_size) <= 0 or int(pool_size) > 100:
+            warnings.warn("pool_size 有效范围为(0, 100]，设置的{}将不会生效".format(pool_size))
+        else:
+            conf["pool_size"] = pool_size
+
+    compressor = kwargs.pop("compressor", None)
+    if compressor is None:
+        compressor = conf.get("compressor", None)
+    else:
+        conf["compressor"] = compressor
+
+    page_size = kwargs.pop("page_size", None)
+    if page_size is None:
+        page_size = conf.get("page_size", 100000)
+    else:
+        if int(page_size) <= 0 or int(page_size) > 100000:
+            warnings.warn("page_size 有效范围为(0, 100000]，设置的{}将不会生效".format(page_size))
+        else:
+            conf["page_size"] = page_size
+
+    quote_parallel = kwargs.pop("quote_parallel", None)
+    if quote_parallel is None:
+        quote_parallel = conf.get("quote_parallel", 50)
+    conf["quote_parallel"] = quote_parallel
+
+    quote_format = kwargs.pop("quote_format", None)
+    if quote_format is None:
+        quote_format = conf.get("quote_format", "arrow")
+    if quote_format.lower() in ('a', 'arrow'):
+        quote_format = "Arrow"
+    elif quote_format.lower() in ('p', 'parquet'):
+        quote_format = "Parquet"
+    else:
+        raise ValueError("Unknown quote_format, available values are:a for Arrow,p for Parquet")
+
+    conf["quote_format"] = quote_format
+
+    parallel_mode = kwargs.pop("parallel_mode", None)
+    if parallel_mode is None:
+        parallel_mode = conf.get("parallel_mode", "t")
+    if parallel_mode.lower() in ('t', 'thread'):
+        parallel_mode = "Thread"
+    elif parallel_mode.lower() in ('c', 'coroutine'):
+        parallel_mode = "Coroutine"
+    else:
+        raise ValueError("Unknown quote_format, available values are:t for Thread,c for Coroutine")
+
+    conf["parallel_mode"] = parallel_mode
+
+    config = {
+        "protocol": protocol,
+        "auth": {
+            "username": user,
+            "password": pwd,
+        },
+        "request_timeout": timeout,
+        "connect_timeout": con_timeout,
+        "url": url,
+        "pool_size": pool_size,
+        "compressor": compressor,
+        "page_size": page_size,
+        "quote_format": quote_format,
+    }
+
+    if protocol == "HTTP":
+        from dataquant.utils.connection_pool import ConnectionPool
+        global _CLIENT
+        _CLIENT = _CLIENT = ConnectionPool(config)
+    else:
+        raise RuntimeError("传输协议无效，got protocol[{}]".format(protocol))
+
+    # 保存配置文件
+    if kwargs_num > 0:
+        write_config({'system': conf})
+    #
+    from dataquant.apis.base.api import load_conf as load_base_conf
+    from dataquant.apis.quote.api import load_conf as load_quote_conf
+    load_base_conf()
+    load_quote_conf()
+    return
+
+
+def environ():
+    """
+    获取环境配置信息
+    :return:
+    """
+    global _CLIENT
+    if _CLIENT:
+        return _CLIENT.config
+
+    return {}

dataquant/utils/common.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+from enum import Enum
+
+# 服务交互字段定义
+ERROR_NO = "error_no"    # 错误代码域
+ERROR_INFO = "error_info"  # 错误信息域
+DATA = "data"    # 数据域
+
+
+class ConnectionStatus(Enum):                 #连接代码
+    Disconnected = 0x0000       # 未连接
+    Connecting = 0x0001,        # 正在连接
+    Connected = 0x0002,         # 已连接
+    SafeConnecting = 0x0004,    # 正在建立安全连接
+    SafeConnected = 0x0008,     # 已建立安全连接
+    Registering = 0x0010,       # 正注册
+    Registered = 0x0020,        # 已注册
+    Rejected = 0x0040           # 被拒绝，将被关闭
+
+
+class Protocol(Enum):                 #连接类型
+    HTTP = "HttpConnection"
+    TCP = "TCPConnection"

dataquant/utils/config.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+import os
+import yaml
+
+from dataquant.utils.decorators import lru_cache
+
+
+def load_yml(path):
+    with open(path, encoding='utf-8') as pf:
+        return yaml.safe_load(pf)
+
+
+def save_yml(full_path, _yml):
+    with open(full_path, encoding='utf-8', mode='w') as pf:
+        return yaml.safe_dump(_yml, pf)
+
+
+@lru_cache(128)
+def read_config():
+    path = os.path.join(os.path.dirname(__file__), ".", "config.yml")
+    return load_yml(path)
+
+
+def write_config(_config):
+    conf = read_config()['system']
+    import operator
+    if operator.ne(_config, conf):
+        path = os.path.join(os.path.dirname(__file__), ".", "config.yml")
+        return save_yml(path, _config)

dataquant/utils/config.yml

@@ -0,0 +1,30 @@
+system:
+  bind_method: socket
+  bind_port: 47317
+  compressor: null
+  connect_timeout: 5
+  page_size: 990000
+  parallel_mode: Thread
+  password: 19515beb
+  pool_size: 10
+  protocol: HTTP
+  quote_format: Arrow
+  quote_hf_bond_sort: scr_num,date_time
+  quote_hf_fund_sort: scr_num,date_time
+  quote_hf_future_sort: scr_num,date_time
+  quote_hf_stock_sort: scr_num,date_time
+  quote_hf_hkstock_sort: scr_num,date_time
+  quote_kd_bond_sort: scr_num,date_time
+  quote_kd_fund_sort: scr_num,date_time
+  quote_kd_future_sort: scr_num,date_time
+  quote_kd_stock_sort: scr_num,date_time
+  quote_km_bond_sort: scr_num,date_time,min_time
+  quote_km_fund_sort: scr_num,date_time,min_time
+  quote_km_future_sort: scr_num,date_time,min_time
+  quote_km_stock_sort: scr_num,date_time,min_time
+  quote_parallel: 20
+  request_timeout: 300
+  uote_hf_bond_sort: scr_num,date_time
+  uote_hf_fund_sort: scr_num,date_time
+  url: http://10.56.37.27:8082/bdataspi/spi/1.0/dataservice
+  username: license