hikyuu 2.7.0__py3-none-manylinux2014_aarch64.whl → 2.7.5__py3-none-manylinux2014_aarch64.whl

hikyuu/data/clickhouse_upgrade/0001.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `hku_base`.`stocktypeinfo` UPDATE `minTradeNumber`=100 WHERE `type`=5 AND `description`='ETF';
+ALTER TABLE `hku_base`.`version` UPDATE `version`=1 WHERE `id`=0;

hikyuu/data/clickhouse_upgrade/0002.sql

@@ -0,0 +1,9 @@
+ALTER TABLE `hku_base`.`stocktypeinfo` 
+UPDATE `minTradeNumber` = CASE
+    WHEN `type` = 9 AND `description` = '科创板' THEN 200
+    WHEN `type` = 11 AND `description` = '北交所' THEN 100
+    ELSE `minTradeNumber`
+END
+WHERE (`type` = 9 AND `description` = '科创板') 
+   OR (`type` = 11 AND `description` = '北交所');
+ALTER TABLE `hku_base`.`version` UPDATE `version`=2 WHERE `id`=0;

hikyuu/data/common_clickhouse.py

@@ -69,11 +69,9 @@ def create_database(connect):
              and int(x.stem) > db_version and not x.is_dir()]
     files.sort()
     for file in files:
-        print(file)
         sql = file.read_text(encoding='utf8')
         statements = sql.split(';')
         for stmt in statements:
-            print(stmt)
             stmt = stmt.strip()
             if stmt:  # 跳过空语句
                 try:
@@ -163,7 +161,7 @@ def get_last_krecord(connect, tablename):
         # hku_info(f"{tablename} {a}")
         if not a:
             return None
-        return (Datetime.from_timestamp_utc(a[0][0]*1000000).ymdhm, a[0][1]*0.001, a[0][2]*0.001, a[0][3]*0.001, a[0][4]*0.001, a[0][5]*0.001, a[0][6]*0.001)
+        return (Datetime.from_timestamp_utc(a[0][0]*1000000).ymdhm, a[0][1]*0.001, a[0][2]*0.001, a[0][3]*0.001, a[0][4]*0.001, float(a[0][5]), a[0][6]*0.001)
     except:
         return None
 

hikyuu/data/common_mysql.py

@@ -502,7 +502,7 @@ def update_extern_data(connect, market, code, data_type):
         if insert_buffer:
             cur = connect.cursor()
             cur.executemany(
-                "insert into {} (date, open, high, low, close, amount, count) \
+                "replace into {} (date, open, high, low, close, amount, count) \
                  values (%s, %s, %s, %s, %s, %s, %s)".format(index_table), insert_buffer
             )
             connect.commit()

hikyuu/data/download_block.py

@@ -333,7 +333,7 @@ def download_all_zsbk_info():
 
 
 def download_block_info():
-    down_em_all_hybk_info()
+    # down_em_all_hybk_info()
     # down_em_all_dybk_info()
     # down_em_all_gnbk_info()
     download_all_zsbk_info()

hikyuu/data/em_block_to_mysql.py

@@ -22,10 +22,22 @@ def em_import_block_to_mysql(connect):
 
     hku_info("更新数据库")
     cur = connect.cursor()
-    sql = f"delete from hku_base.block where category in {tuple(blks.keys())}"
-    cur.execute(sql)
-    sql = f"delete from hku_base.BlockIndex where category in {tuple(blks.keys())}"
-    cur.execute(sql)
+    
+    # 构建参数化的DELETE语句以防止SQL注入
+    categories = tuple(blks.keys())
+    if len(categories) == 1:
+        # 处理只有一个类别的特殊情况
+        placeholders = '%s'
+        params = categories
+    else:
+        placeholders = ','.join(['%s'] * len(categories))  # 创建相应数量的占位符
+        params = categories
+    
+    sql = f"delete from hku_base.block where category in ({placeholders})"
+    cur.execute(sql, params)
+    
+    sql = f"delete from hku_base.BlockIndex where category in ({placeholders})"
+    cur.execute(sql, params)
 
     insert_records = []
 

hikyuu/data/em_block_to_sqlite.py

@@ -23,10 +23,22 @@ def em_import_block_to_sqlite(connect):
 
     hku_info("更新数据库")
     cur = connect.cursor()
-    sql = f"delete from block where category in {tuple(blks.keys())}"
-    cur.execute(sql)
-    sql = f"delete from BlockIndex where category in {tuple(blks.keys())}"
-    cur.execute(sql)
+    
+    # 构建参数化的DELETE语句以防止SQL注入
+    categories = tuple(blks.keys())
+    if len(categories) == 1:
+        # 处理只有一个类别的特殊情况
+        placeholders = '?'
+        params = categories
+    else:
+        placeholders = ','.join(['?'] * len(categories))  # 创建相应数量的占位符
+        params = categories
+    
+    sql = f"delete from block where category in ({placeholders})"
+    cur.execute(sql, params)
+    
+    sql = f"delete from BlockIndex where category in ({placeholders})"
+    cur.execute(sql, params)
 
     insert_records = []
 

hikyuu/data/hku_config_template.py

@@ -26,7 +26,9 @@ hdf5_template = """
 [hikyuu]
 tmpdir = {dir}/tmp
 datadir = {dir}
+reload_time = {reload_time}
 quotation_server = {quotation_server}
+lazy_preload = {lazy_preload}
 
 [block]
 type = sqlite3
@@ -52,6 +54,8 @@ min15 = {min15}
 min30 = {min30}
 min60 = {min60}
 hour2= {hour2}
+timeline = {timeline}
+trans = {trans}
 day_max = {day_max}
 week_max = {week_max}
 month_max = {month_max}
@@ -64,6 +68,8 @@ min15_max = {min15_max}
 min30_max = {min30_max}
 min60_max = {min60_max}
 hour2_max = {hour2_max}
+timeline_max = {timeline_max}
+trans_max = {trans_max}
 
 [baseinfo]
 type = sqlite3
@@ -94,7 +100,9 @@ mysql_template = """
 [hikyuu]
 tmpdir = {dir}
 datadir = {dir}
+reload_time = {reload_time}
 quotation_server = {quotation_server}
+lazy_preload = {lazy_preload}
 
 [block]
 type = mysql
@@ -116,6 +124,8 @@ min15 = {min15}
 min30 = {min30}
 min60 = {min60}
 hour2 = {hour2}
+timeline = {timeline}
+trans = {trans}
 day_max = {day_max}
 week_max = {week_max}
 month_max = {month_max}
@@ -128,6 +138,8 @@ min15_max = {min15_max}
 min30_max = {min30_max}
 min60_max = {min60_max}
 hour2_max = {hour2_max}
+timeline_max = {timeline_max}
+trans_max = {trans_max}
 
 [baseinfo]
 type = mysql
@@ -149,7 +161,9 @@ clickhouse_template = """
 [hikyuu]
 tmpdir = {dir}
 datadir = {dir}
+reload_time = {reload_time}
 quotation_server = {quotation_server}
+lazy_preload = {lazy_preload}
 
 [block]
 type = clickhouse
@@ -171,6 +185,8 @@ min15 = {min15}
 min30 = {min30}
 min60 = {min60}
 hour2 = {hour2}
+timeline = {timeline}
+trans = {trans}
 day_max = {day_max}
 week_max = {week_max}
 month_max = {month_max}
@@ -183,6 +199,8 @@ min15_max = {min15_max}
 min30_max = {min30_max}
 min60_max = {min60_max}
 hour2_max = {hour2_max}
+timeline_max = {timeline_max}
+trans_max = {trans_max}
 
 [baseinfo]
 type = clickhouse
@@ -269,6 +287,9 @@ phase1_end = 11:35:00
 phase2_start = 12:00:00
 phase2_end = 15:05:00
 
+[lazy_preload]
+enable = False
+
 [preload]
 day = True
 week = True
@@ -282,6 +303,8 @@ min15 = False
 min30 = False
 min60 = False
 hour2 = False
+timeline = False
+trans = False
 day_max = 100000
 week_max = 100000
 month_max = 100000
@@ -294,6 +317,8 @@ min15_max = 5120
 min30_max = 5120
 min60_max = 5120
 hour2_max = 5120
+timeline_max = 5120
+trans_max = 5120
 """
 
 
@@ -304,14 +329,16 @@ def generate_default_config():
     from hikyuu.data.hku_config_template import hdf5_template
     user_dir = os.path.expanduser('~')
     data_dir = "c:\\stock" if sys.platform == 'win32' else f"{user_dir}/stock"
-    hdf5_config = hdf5_template.format(dir=data_dir, quotation_server='ipc:///tmp/hikyuu_real.ipc',
-                                       day=True, week=False,
+    hdf5_config = hdf5_template.format(dir=data_dir, reload_time="00:00", quotation_server='ipc:///tmp/hikyuu_real.ipc',
+                                       lazy_preload=False, day=True, week=False,
                                        month=False, quarter=False, halfyear=False, year=False,
                                        min1=False, min5=False, min15=False, min30=False,
-                                       min60=False, hour2=False, day_max=100000, week_max=100000,
+                                       min60=False, hour2=False, timeline=False, trans=False,
+                                       day_max=100000, week_max=100000,
                                        month_max=100000, quarter_max=100000, halfyear_max=100000,
                                        year_max=100000, min1_max=5120, min5_max=5120, min15_max=5120,
-                                       min30_max=5120, min60_max=5120, hour2_max=5120)
+                                       min30_max=5120, min60_max=5120, hour2_max=5120,
+                                       timeline_max=5120, trans_max=5120)
     config_dir = f"{user_dir}/.hikyuu"
     if not os.path.lexists(config_dir):
         os.makedirs(config_dir)

hikyuu/data/mysql_upgrade/0029.sql

@@ -0,0 +1,2 @@
+update `hku_base`.`stocktypeinfo` set `minTradeNumber`=100 where `type`=5 and `description`='ETF';
+UPDATE `hku_base`.`version` set `version` = 29;

hikyuu/data/mysql_upgrade/0030.sql

@@ -0,0 +1,3 @@
+update `hku_base`.`stocktypeinfo` set `minTradeNumber`=200 where `type`=9 and `description`='科创板';
+update `hku_base`.`stocktypeinfo` set `minTradeNumber`=100 where `type`=11 and `description`='北交所';
+UPDATE `hku_base`.`version` set `version` = 30;

hikyuu/data/pytdx_to_clickhouse.py

@@ -23,13 +23,15 @@
 # SOFTWARE.
 
 import sys
+import os
 import math
 import datetime
 from pytdx.hq import TDXParams
+from configparser import ConfigParser
 
 from hikyuu.util import hku_error, hku_debug, hku_run_ignore_exception
 
-from hikyuu import Datetime, roundEx
+from hikyuu import Datetime, roundEx, is_valid_license, KDataToClickHouseImporter
 from hikyuu.data.common import *
 from hikyuu.data.common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from hikyuu.data.common_clickhouse import (
@@ -134,6 +136,9 @@ def import_stock_name(connect, api, market, quotations=None):
         if code not in deSet:
             newStockDict[code] = stock["name"]
 
+    # 号码前缀
+    codepre_list = get_codepre_list(connect, market, quotations)
+
     stktype_list = get_stktype_list(quotations)
     stktype_list = list(stktype_list)
     stktype_list.remove(STOCKTYPE.INDEX)  # 移除指数类型
@@ -149,23 +154,29 @@ def import_stock_name(connect, api, market, quotations=None):
 
         oldname, oldtype, oldvalid, oldstartDate, oldendDate = oldstock[2], oldstock[3], oldstock[4], oldstock[5], oldstock[6]
 
+        # 检测是否类型发生变化
+        newtype = oldtype
+        for codepre in codepre_list:
+            length = len(codepre[0])
+            if oldcode[:length] == codepre[0]:
+                if oldtype != codepre[1]:
+                    newtype = codepre[1]
+                break
+
         # 新的代码表中无此股票，则置为无效
-        # if (oldvalid == 1) and (oldcode not in newStockDict):
         if (oldvalid == 1) and ((oldcode not in newStockDict) or oldcode in deSet):
             sql = f"delete from hku_base.stock where market='{market}' and code='{oldcode}'"
             connect.command(sql)
-            insert_records.append((market, oldcode, oldname, oldtype, 0, oldstartDate, oldendDate))
+            insert_records.append((market, oldcode, oldname, newtype, 0, oldstartDate, oldendDate))
 
         # 股票名称发生变化，更新股票名称;如果原无效，则置为有效
         if oldcode in newStockDict:
-            if oldname != newStockDict[oldcode] or oldvalid == 0:
+            if oldname != newStockDict[oldcode] or oldvalid == 0 or oldtype != newtype:
                 sql = f"delete from hku_base.stock where market='{market}' and code='{oldcode}'"
                 connect.command(sql)
-                insert_records.append((market, oldcode, newStockDict[oldcode], oldtype, 1, oldstartDate, 99999999))
+                insert_records.append((market, oldcode, newStockDict[oldcode], newtype, 1, oldstartDate, 99999999))
 
     # 处理新出现的股票
-    codepre_list = get_codepre_list(connect, market, quotations)
-
     today = datetime.date.today()
     today = today.year * 10000 + today.month * 100 + today.day
     count = 0
@@ -274,10 +285,10 @@ def import_one_stock_data(
             today.year * 10000 + today.month * 100 + today.day
         ) * 10000 + 1500
     else:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     if today_datetime <= last_datetime:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     get_bars = (
         api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
@@ -312,27 +323,27 @@ def import_one_stock_data(
                 if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord[1]}, bar: {bar['open']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[2] - bar["high"]) / last_krecord[2] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord[2]}, bar: {bar['high']}")
-                    return 0
-                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01:
+                    return (0, False,  Datetime(last_datetime))
+                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01 and abs(last_krecord[3] - bar["low"]) > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord[3]}, bar: {bar['low']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[4] - bar["close"]) / last_krecord[4] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord[4]}, bar: {bar['close']}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]*0.001) / last_krecord[5] > 0.1:
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]) > 10000:
                     hku_error(
-                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[6] != 0.0 and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']}")
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[6] != 0.0 and last_krecord[6] != 0.0 and abs(last_krecord[6] - bar["vol"]) > 10000:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 continue
 
             if (
@@ -365,7 +376,7 @@ def import_one_stock_data(
                                            data=buf)
         connect.insert(context=ic, settings={"prefer_warmed_unmerged_parts_seconds": 86400})
 
-    return len(buf)
+    return (len(buf), True, Datetime(last_datetime))
 
 
 def update_stock_info(connect, market):
@@ -409,7 +420,7 @@ def clear_extern_data(connect, market, data_type):
         index_list = ('min15', 'min30', 'min60', 'hour2')
         lastdate = connect.command(
             f"select toInt32(max(date)) from hku_data.min5_k where market='SH' and code='000001'")
-        lastdate = Datetime.from_timestamp_utc(lastdate*1000000).start_of_day()
+        lastdate = Datetime.from_timestamp_utc(int(lastdate*1000000)).start_of_day()
         last_timestamp = Datetime(lastdate).timestamp_utc()//1000000
         for index_type in index_list:
             sql = f"delete from hku_data.{index_type}_k where market='{market}' and date>={last_timestamp}"
@@ -420,6 +431,16 @@ def clear_extern_data(connect, market, data_type):
         hku_info(f"清理 {market} {data_type} 线扩展数据完毕")
 
 
+@hku_catch(ret=None)
+def get_clickhouse_importer():
+    filename = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini'
+    config = ConfigParser()
+    config.read(filename, encoding='utf-8')
+    importer = KDataToClickHouseImporter()
+    return importer if importer.set_config(config.get("kdata", "host"), config.getint("kdata", "port", fallback=9000),
+                                           config.get("kdata", "usr"), config.get("kdata", "pwd")) else None
+
+
 @hku_catch(trace=True, re_raise=True)
 def import_data(
     connect,
@@ -460,6 +481,9 @@ def import_data(
             for index_type in index_list:
                 update_data[index_type] = []
 
+    failed_limit = 20
+    failed_count = 0
+    failed_list = []
     total = len(stock_list)
     # market, code, valid, type
     for i, stock in enumerate(stock_list):
@@ -468,9 +492,15 @@ def import_data(
                 progress(i, total)
             continue
 
-        this_count = import_one_stock_data(
+        this_count, success, lastdate = import_one_stock_data(
             connect, api, market, ktype, stock, startDate
         )
+        if not success:
+            failed_count += 1
+            failed_list.append((market, stock[1], lastdate))
+        if failed_count >= failed_limit:
+            # hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+            break
         add_record_count += this_count
         if ktype in ("DAY", "5MIN"):
             if ktype == "DAY":
@@ -490,6 +520,9 @@ def import_data(
         if progress:
             progress(i, total)
 
+    if total > 0 and progress:
+        progress(total, total)
+
     if ktype in ("DAY", "5MIN"):
         for index_type in index_list:
             if len(update_data[index_type]) > 0:
@@ -501,6 +534,25 @@ def import_data(
                 update_data[index_type].clear()
         update_data.clear()
 
+    if 0 < failed_count < failed_limit and is_valid_license():
+        # 删除最后记录
+        ktype_dict = {
+            'DAY': 'DAY',
+            '1MIN': 'MIN',
+            '5MIN': 'MIN5'
+        }
+        nktype = ktype_dict[ktype]
+        ch_importer = get_clickhouse_importer()
+        if ch_importer is not None:
+            for r in failed_list:
+                hku_warn(f"清理 {r[0]} {r[1]} {nktype} {r[2].start_of_day()}")
+                ch_importer.remove(r[0], r[1], nktype, r[2].start_of_day())
+            hku_warn(f"已清理 {market} {failed_count} 个失败股票的最后记录，建议重新导入")
+
+    if failed_count >= failed_limit:
+        hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+        return add_record_count
+
     cur_year = Datetime.today().year
     if ktype == "DAY":
         update_stock_info(connect, market)
@@ -592,15 +644,16 @@ def import_on_stock_trans(connect, api, market, stock_record, max_days):
                 if second > 59:
                     continue
 
-                trans_buf.append(
-                    (
-                        market, code,
-                        Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
-                        int(roundEx(record["price"], 3) * 1000.0),
-                        int(record["vol"]),
-                        record["buyorsell"],
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    trans_buf.append(
+                        (
+                            market, code,
+                            Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
+                            int(roundEx(record["price"], 3) * 1000.0),
+                            int(record["vol"]),
+                            record["buyorsell"],
+                        )
                     )
-                )
             except Exception as e:
                 hku_error("Failed trans to record! {}", e)
 
@@ -701,8 +754,9 @@ def import_on_stock_time(connect, api, market, stock_record, max_days):
             elif time == 1360:
                 time = 1400
             try:
-                time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
-                                ticks, int(roundEx(record['price'], 3) * 1000.0), int(record['vol'])))
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
+                                     ticks, int(roundEx(record['price'], 3) * 1000.0), int(record['vol'])))
                 time += 1
             except Exception as e:
                 hku_error("Failed trans record {}! {}".format(record, e))