hikyuu 2.6.8.3__py3-none-manylinux2014_aarch64.whl → 2.7.3__py3-none-manylinux2014_aarch64.whl

hikyuu/data/em_block_to_sqlite.py

@@ -4,97 +4,51 @@
 # Create on: 20240102
 #    Author: fasiondog
 
-from concurrent.futures import ThreadPoolExecutor
-from hikyuu.data.common import MARKET, get_stk_code_name_list
+
 from hikyuu.util import *
 from hikyuu.fetcher.stock.zh_block_em import *
+from hikyuu.data.download_block import *
 
 
 @spend_time
-def em_import_block_to_sqlite(connect, code_market_dict, categorys=('行业板块', '概念板块', '地域板块', '指数板块')):
-    all_block_info = {}
-
-    with ThreadPoolExecutor(4) as executor:
-        if '行业板块' in categorys:
-            t1 = executor.submit(get_all_hybk_info, code_market_dict)
-
-        if '概念板块' in categorys:
-            t2 = executor.submit(get_all_gnbk_info, code_market_dict)
-
-        if '地域板块' in categorys:
-            t3 = executor.submit(get_all_dybk_info, code_market_dict)
-
-        if '指数板块' in categorys:
-            t4 = executor.submit(get_all_zsbk_info, code_market_dict)
-
-        success_fetch_hy = False
-        if '行业板块' in categorys:
-            x = t1.result()
-            hku_info("获取行业板块信息完毕")
-            if x:
-                all_block_info["行业板块"] = x
-                success_fetch_hy = True
-
-        success_fetch_gn = False
-        if '概念板块' in categorys:
-            x = t2.result()
-            hku_info("获取概念板块信息完毕")
-            if x:
-                all_block_info["概念板块"] = x
-                success_fetch_gn = True
-
-        success_fetch_dy = False
-        if '地域板块' in categorys:
-            x = t3.result()
-            hku_info("获取地域板块信息完毕")
-            if x:
-                all_block_info["地域板块"] = x
-                success_fetch_dy = True
-
-        success_fetch_zs = False
-        if '指数板块' in categorys:
-            x = t4.result()
-            hku_info("获取指数板块信息完毕")
-            if x:
-                all_block_info["指数板块"] = x
-                success_fetch_zs = True
-
-    blks = []
-    if success_fetch_hy:
-        blks.append('行业板块')
-    if success_fetch_gn:
-        blks.append('概念板块')
-    if success_fetch_dy:
-        blks.append('地域板块')
-    if success_fetch_zs:
-        blks.append('指数板块')
-
-    print(blks)
+def em_import_block_to_sqlite(connect):
+    download_block_info()
+
+    blocks_info = read_block_from_path()
+    blks = blocks_info['block']
+    blks_info = blocks_info['block_info']
 
     if not blks:
-        return
+        return 0
 
     hku_info("更新数据库")
     cur = connect.cursor()
-    if len(blks) == 1:
-        sql = f"delete from block where category in ('{blks[0]}')"
-    else:
-        sql = f"delete from block where category in {tuple(blks)}"
-    hku_info(sql)
+    sql = f"delete from block where category in {tuple(blks.keys())}"
+    cur.execute(sql)
+    sql = f"delete from BlockIndex where category in {tuple(blks.keys())}"
     cur.execute(sql)
 
     insert_records = []
 
-    for category in all_block_info:
-        for name in all_block_info[category]:
-            for code in all_block_info[category][name]:
+    for category, blocks in blks.items():
+        for name, codes in blocks.items():
+            for code in codes:
                 insert_records.append((category, name, code))
 
     if insert_records:
         sql = "insert into block (category, name, market_code) values (?,?,?)"
-        hku_info(f"insert block records: {len(insert_records)}")
         cur.executemany(sql, insert_records)
 
+    index_records = []
+    for category, blocks in blks_info.items():
+        for name, index_code in blocks.items():
+            if len(index_code) == 8:
+                index_records.append((category, name, index_code))
+    if index_records:
+        sql = "insert into BlockIndex (category, name, market_code) values (?,?,?)"
+        cur.executemany(sql, index_records)
+        hku_info(f"insert block index records: {len(index_records)}")
+
     connect.commit()
     cur.close()
     return len(insert_records)
@@ -111,12 +65,6 @@ if __name__ == "__main__":
     connect = sqlite3.connect(dest_dir + "/stock.db")
     create_database(connect)
 
-    x = get_stk_code_name_list(MARKET.SH)
-    code_market_dict = {}
-    for v in x:
-        code_market_dict[v["code"]] = MARKET.SH
-    # print(code_market_dict)
-
-    em_import_block_to_sqlite(connect, code_market_dict, categorys=('地域板块',))
+    em_import_block_to_sqlite(connect)
 
     connect.close()

hikyuu/data/hku_config_template.py

@@ -26,7 +26,9 @@ hdf5_template = """
 [hikyuu]
 tmpdir = {dir}/tmp
 datadir = {dir}
+reload_time = {reload_time}
 quotation_server = {quotation_server}
+lazy_preload = {lazy_preload}
 
 [block]
 type = sqlite3
@@ -52,6 +54,8 @@ min15 = {min15}
 min30 = {min30}
 min60 = {min60}
 hour2= {hour2}
+timeline = {timeline}
+trans = {trans}
 day_max = {day_max}
 week_max = {week_max}
 month_max = {month_max}
@@ -64,6 +68,8 @@ min15_max = {min15_max}
 min30_max = {min30_max}
 min60_max = {min60_max}
 hour2_max = {hour2_max}
+timeline_max = {timeline_max}
+trans_max = {trans_max}
 
 [baseinfo]
 type = sqlite3
@@ -94,7 +100,9 @@ mysql_template = """
 [hikyuu]
 tmpdir = {dir}
 datadir = {dir}
+reload_time = {reload_time}
 quotation_server = {quotation_server}
+lazy_preload = {lazy_preload}
 
 [block]
 type = mysql
@@ -116,6 +124,8 @@ min15 = {min15}
 min30 = {min30}
 min60 = {min60}
 hour2 = {hour2}
+timeline = {timeline}
+trans = {trans}
 day_max = {day_max}
 week_max = {week_max}
 month_max = {month_max}
@@ -128,6 +138,8 @@ min15_max = {min15_max}
 min30_max = {min30_max}
 min60_max = {min60_max}
 hour2_max = {hour2_max}
+timeline_max = {timeline_max}
+trans_max = {trans_max}
 
 [baseinfo]
 type = mysql
@@ -149,7 +161,9 @@ clickhouse_template = """
 [hikyuu]
 tmpdir = {dir}
 datadir = {dir}
+reload_time = {reload_time}
 quotation_server = {quotation_server}
+lazy_preload = {lazy_preload}
 
 [block]
 type = clickhouse
@@ -171,6 +185,8 @@ min15 = {min15}
 min30 = {min30}
 min60 = {min60}
 hour2 = {hour2}
+timeline = {timeline}
+trans = {trans}
 day_max = {day_max}
 week_max = {week_max}
 month_max = {month_max}
@@ -183,6 +199,8 @@ min15_max = {min15_max}
 min30_max = {min30_max}
 min60_max = {min60_max}
 hour2_max = {hour2_max}
+timeline_max = {timeline_max}
+trans_max = {trans_max}
 
 [baseinfo]
 type = clickhouse
@@ -210,8 +228,8 @@ future = False
 day = True
 min = True
 min5 = True
-trans = True
-time = True
+trans = False
+time = False
 day_start_date = 1990-12-19
 min_start_date = 2023-09-19
 min5_start_date = 2023-09-19
@@ -251,7 +269,7 @@ enable = False
 enable = False
 ;tmpdir = D:/stock
 ;host = 127.0.0.1
-:http_port = 8123
+;http_port = 8123
 ;port = 9000
 ;usr = root
 ;pwd = 
@@ -269,6 +287,9 @@ phase1_end = 11:35:00
 phase2_start = 12:00:00
 phase2_end = 15:05:00
 
+[lazy_preload]
+enable = False
+
 [preload]
 day = True
 week = True
@@ -282,6 +303,8 @@ min15 = False
 min30 = False
 min60 = False
 hour2 = False
+timeline = False
+trans = False
 day_max = 100000
 week_max = 100000
 month_max = 100000
@@ -294,6 +317,8 @@ min15_max = 5120
 min30_max = 5120
 min60_max = 5120
 hour2_max = 5120
+timeline_max = 5120
+trans_max = 5120
 """
 
 
@@ -304,14 +329,16 @@ def generate_default_config():
     from hikyuu.data.hku_config_template import hdf5_template
     user_dir = os.path.expanduser('~')
     data_dir = "c:\\stock" if sys.platform == 'win32' else f"{user_dir}/stock"
-    hdf5_config = hdf5_template.format(dir=data_dir, quotation_server='ipc:///tmp/hikyuu_real.ipc',
+    hdf5_config = hdf5_template.format(dir=data_dir, reload_time="00:00", quotation_server='ipc:///tmp/hikyuu_real.ipc',
                                        day=True, week=False,
                                        month=False, quarter=False, halfyear=False, year=False,
                                        min1=False, min5=False, min15=False, min30=False,
-                                       min60=False, hour2=False, day_max=100000, week_max=100000,
+                                       min60=False, hour2=False, timeline=False, trans=False,
+                                       day_max=100000, week_max=100000,
                                        month_max=100000, quarter_max=100000, halfyear_max=100000,
                                        year_max=100000, min1_max=5120, min5_max=5120, min15_max=5120,
-                                       min30_max=5120, min60_max=5120, hour2_max=5120)
+                                       min30_max=5120, min60_max=5120, hour2_max=5120,
+                                       timeline_max=5120, trans_max=5120)
     config_dir = f"{user_dir}/.hikyuu"
     if not os.path.lexists(config_dir):
         os.makedirs(config_dir)

hikyuu/data/mysql_upgrade/0029.sql

@@ -0,0 +1,2 @@
+update `hku_base`.`stocktypeinfo` set `minTradeNumber`=100 where `type`=5 and `description`='ETF';
+UPDATE `hku_base`.`version` set `version` = 29;

hikyuu/data/pytdx_to_clickhouse.py

@@ -23,13 +23,15 @@
 # SOFTWARE.
 
 import sys
+import os
 import math
 import datetime
 from pytdx.hq import TDXParams
+from configparser import ConfigParser
 
 from hikyuu.util import hku_error, hku_debug, hku_run_ignore_exception
 
-from hikyuu import Datetime
+from hikyuu import Datetime, roundEx, is_valid_license, KDataToClickHouseImporter
 from hikyuu.data.common import *
 from hikyuu.data.common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from hikyuu.data.common_clickhouse import (
@@ -134,6 +136,9 @@ def import_stock_name(connect, api, market, quotations=None):
         if code not in deSet:
             newStockDict[code] = stock["name"]
 
+    # 号码前缀
+    codepre_list = get_codepre_list(connect, market, quotations)
+
     stktype_list = get_stktype_list(quotations)
     stktype_list = list(stktype_list)
     stktype_list.remove(STOCKTYPE.INDEX)  # 移除指数类型
@@ -149,23 +154,29 @@ def import_stock_name(connect, api, market, quotations=None):
 
         oldname, oldtype, oldvalid, oldstartDate, oldendDate = oldstock[2], oldstock[3], oldstock[4], oldstock[5], oldstock[6]
 
+        # 检测是否类型发生变化
+        newtype = oldtype
+        for codepre in codepre_list:
+            length = len(codepre[0])
+            if oldcode[:length] == codepre[0]:
+                if oldtype != codepre[1]:
+                    newtype = codepre[1]
+                break
+
         # 新的代码表中无此股票，则置为无效
-        # if (oldvalid == 1) and (oldcode not in newStockDict):
         if (oldvalid == 1) and ((oldcode not in newStockDict) or oldcode in deSet):
             sql = f"delete from hku_base.stock where market='{market}' and code='{oldcode}'"
             connect.command(sql)
-            insert_records.append((market, oldcode, oldname, oldtype, 0, oldstartDate, oldendDate))
+            insert_records.append((market, oldcode, oldname, newtype, 0, oldstartDate, oldendDate))
 
         # 股票名称发生变化，更新股票名称;如果原无效，则置为有效
         if oldcode in newStockDict:
-            if oldname != newStockDict[oldcode] or oldvalid == 0:
+            if oldname != newStockDict[oldcode] or oldvalid == 0 or oldtype != newtype:
                 sql = f"delete from hku_base.stock where market='{market}' and code='{oldcode}'"
                 connect.command(sql)
-                insert_records.append((market, oldcode, newStockDict[oldcode], oldtype, 1, oldstartDate, 99999999))
+                insert_records.append((market, oldcode, newStockDict[oldcode], newtype, 1, oldstartDate, 99999999))
 
     # 处理新出现的股票
-    codepre_list = get_codepre_list(connect, market, quotations)
-
     today = datetime.date.today()
     today = today.year * 10000 + today.month * 100 + today.day
     count = 0
@@ -274,10 +285,10 @@ def import_one_stock_data(
             today.year * 10000 + today.month * 100 + today.day
         ) * 10000 + 1500
     else:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     if today_datetime <= last_datetime:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     get_bars = (
         api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
@@ -304,31 +315,35 @@ def import_one_stock_data(
                 hku_error("Failed translate datetime: {}, from {}! {}".format(bar, api.ip, e))
                 continue
 
+            if 'open' not in bar or 'high' not in bar or 'low' not in bar or 'close' not in bar or 'vol' not in bar or 'amount' not in bar:
+                hku_warn(f"Ignore invalid bar: {bar_datetime}, missing fields!")
+                continue
+
             if last_krecord is not None and bar_datetime == last_datetime:
                 if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord[1]}, bar: {bar['open']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[2] - bar["high"]) / last_krecord[2] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord[2]}, bar: {bar['high']}")
-                    return 0
-                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01:
+                    return (0, False,  Datetime(last_datetime))
+                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01 and abs(last_krecord[3] - bar["low"]) > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord[3]}, bar: {bar['low']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[4] - bar["close"]) / last_krecord[4] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord[4]}, bar: {bar['close']}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]*0.001) / last_krecord[5] > 0.1:
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]) > 10000:
                     hku_error(
-                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']}")
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[6] != 0.0 and last_krecord[6] != 0.0 and abs(last_krecord[6] - bar["vol"]) > 10000:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 continue
 
             if (
@@ -343,12 +358,12 @@ def import_one_stock_data(
                         (
                             table[1], table[2],
                             Datetime(bar_datetime).timestamp_utc()//1000000,
-                            bar["open"],
-                            bar["high"],
-                            bar["low"],
-                            bar["close"],
-                            bar["amount"] * 0.001,
-                            bar["vol"]
+                            int(roundEx(bar["open"], 3)*1000),
+                            int(roundEx(bar["high"], 3)*1000),
+                            int(roundEx(bar["low"], 3)*1000),
+                            int(roundEx(bar["close"], 3)*1000),
+                            int(roundEx(bar["amount"], 0)),
+                            int(roundEx(bar["vol"], 3)*1000)
                             # bar['vol'] if stktype == 2 else round(bar['vol'] * 0.01)
                         )
                     )
@@ -361,7 +376,7 @@ def import_one_stock_data(
                                            data=buf)
         connect.insert(context=ic, settings={"prefer_warmed_unmerged_parts_seconds": 86400})
 
-    return len(buf)
+    return (len(buf), True, Datetime(last_datetime))
 
 
 def update_stock_info(connect, market):
@@ -405,7 +420,7 @@ def clear_extern_data(connect, market, data_type):
         index_list = ('min15', 'min30', 'min60', 'hour2')
         lastdate = connect.command(
             f"select toInt32(max(date)) from hku_data.min5_k where market='SH' and code='000001'")
-        lastdate = Datetime.from_timestamp_utc(lastdate*1000000).start_of_day()
+        lastdate = Datetime.from_timestamp_utc(int(lastdate*1000000)).start_of_day()
         last_timestamp = Datetime(lastdate).timestamp_utc()//1000000
         for index_type in index_list:
             sql = f"delete from hku_data.{index_type}_k where market='{market}' and date>={last_timestamp}"
@@ -416,6 +431,16 @@ def clear_extern_data(connect, market, data_type):
         hku_info(f"清理 {market} {data_type} 线扩展数据完毕")
 
 
+@hku_catch(ret=None)
+def get_clickhouse_importer():
+    filename = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini'
+    config = ConfigParser()
+    config.read(filename, encoding='utf-8')
+    importer = KDataToClickHouseImporter()
+    return importer if importer.set_config(config.get("kdata", "host"), config.getint("kdata", "port", fallback=9000),
+                                           config.get("kdata", "usr"), config.get("kdata", "pwd")) else None
+
+
 @hku_catch(trace=True, re_raise=True)
 def import_data(
     connect,
@@ -456,6 +481,9 @@ def import_data(
             for index_type in index_list:
                 update_data[index_type] = []
 
+    failed_limit = 20
+    failed_count = 0
+    failed_list = []
     total = len(stock_list)
     # market, code, valid, type
     for i, stock in enumerate(stock_list):
@@ -464,9 +492,15 @@ def import_data(
                 progress(i, total)
             continue
 
-        this_count = import_one_stock_data(
+        this_count, success, lastdate = import_one_stock_data(
             connect, api, market, ktype, stock, startDate
         )
+        if not success:
+            failed_count += 1
+            failed_list.append((market, stock[1], lastdate))
+        if failed_count >= failed_limit:
+            # hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+            break
         add_record_count += this_count
         if ktype in ("DAY", "5MIN"):
             if ktype == "DAY":
@@ -486,6 +520,9 @@ def import_data(
         if progress:
             progress(i, total)
 
+    if total > 0 and progress:
+        progress(total, total)
+
     if ktype in ("DAY", "5MIN"):
         for index_type in index_list:
             if len(update_data[index_type]) > 0:
@@ -497,6 +534,25 @@ def import_data(
                 update_data[index_type].clear()
         update_data.clear()
 
+    if 0 < failed_count < failed_limit and is_valid_license():
+        # 删除最后记录
+        ktype_dict = {
+            'DAY': 'DAY',
+            '1MIN': 'MIN',
+            '5MIN': 'MIN5'
+        }
+        nktype = ktype_dict[ktype]
+        ch_importer = get_clickhouse_importer()
+        if ch_importer is not None:
+            for r in failed_list:
+                hku_warn(f"清理 {r[0]} {r[1]} {nktype} {r[2].start_of_day()}")
+                ch_importer.remove(r[0], r[1], nktype, r[2].start_of_day())
+            hku_warn(f"已清理 {market} {failed_count} 个失败股票的最后记录，建议重新导入")
+
+    if failed_count >= failed_limit:
+        hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+        return add_record_count
+
     cur_year = Datetime.today().year
     if ktype == "DAY":
         update_stock_info(connect, market)
@@ -588,15 +644,16 @@ def import_on_stock_trans(connect, api, market, stock_record, max_days):
                 if second > 59:
                     continue
 
-                trans_buf.append(
-                    (
-                        market, code,
-                        Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
-                        record["price"],
-                        record["vol"],
-                        record["buyorsell"],
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    trans_buf.append(
+                        (
+                            market, code,
+                            Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
+                            int(roundEx(record["price"], 3) * 1000.0),
+                            int(record["vol"]),
+                            record["buyorsell"],
+                        )
                     )
-                )
             except Exception as e:
                 hku_error("Failed trans to record! {}", e)
 
@@ -697,8 +754,9 @@ def import_on_stock_time(connect, api, market, stock_record, max_days):
             elif time == 1360:
                 time = 1400
             try:
-                time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
-                                ticks, record['price'], record['vol']))
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
+                                     ticks, int(roundEx(record['price'], 3) * 1000.0), int(record['vol'])))
                 time += 1
             except Exception as e:
                 hku_error("Failed trans record {}! {}".format(record, e))