hikyuu 2.7.0__py3-none-manylinux2014_aarch64.whl → 2.7.3__py3-none-manylinux2014_aarch64.whl

hikyuu/data/pytdx_to_clickhouse.py

@@ -23,13 +23,15 @@
 # SOFTWARE.
 
 import sys
+import os
 import math
 import datetime
 from pytdx.hq import TDXParams
+from configparser import ConfigParser
 
 from hikyuu.util import hku_error, hku_debug, hku_run_ignore_exception
 
-from hikyuu import Datetime, roundEx
+from hikyuu import Datetime, roundEx, is_valid_license, KDataToClickHouseImporter
 from hikyuu.data.common import *
 from hikyuu.data.common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from hikyuu.data.common_clickhouse import (
@@ -134,6 +136,9 @@ def import_stock_name(connect, api, market, quotations=None):
         if code not in deSet:
             newStockDict[code] = stock["name"]
 
+    # 号码前缀
+    codepre_list = get_codepre_list(connect, market, quotations)
+
     stktype_list = get_stktype_list(quotations)
     stktype_list = list(stktype_list)
     stktype_list.remove(STOCKTYPE.INDEX)  # 移除指数类型
@@ -149,23 +154,29 @@ def import_stock_name(connect, api, market, quotations=None):
 
         oldname, oldtype, oldvalid, oldstartDate, oldendDate = oldstock[2], oldstock[3], oldstock[4], oldstock[5], oldstock[6]
 
+        # 检测是否类型发生变化
+        newtype = oldtype
+        for codepre in codepre_list:
+            length = len(codepre[0])
+            if oldcode[:length] == codepre[0]:
+                if oldtype != codepre[1]:
+                    newtype = codepre[1]
+                break
+
         # 新的代码表中无此股票，则置为无效
-        # if (oldvalid == 1) and (oldcode not in newStockDict):
         if (oldvalid == 1) and ((oldcode not in newStockDict) or oldcode in deSet):
             sql = f"delete from hku_base.stock where market='{market}' and code='{oldcode}'"
             connect.command(sql)
-            insert_records.append((market, oldcode, oldname, oldtype, 0, oldstartDate, oldendDate))
+            insert_records.append((market, oldcode, oldname, newtype, 0, oldstartDate, oldendDate))
 
         # 股票名称发生变化，更新股票名称;如果原无效，则置为有效
         if oldcode in newStockDict:
-            if oldname != newStockDict[oldcode] or oldvalid == 0:
+            if oldname != newStockDict[oldcode] or oldvalid == 0 or oldtype != newtype:
                 sql = f"delete from hku_base.stock where market='{market}' and code='{oldcode}'"
                 connect.command(sql)
-                insert_records.append((market, oldcode, newStockDict[oldcode], oldtype, 1, oldstartDate, 99999999))
+                insert_records.append((market, oldcode, newStockDict[oldcode], newtype, 1, oldstartDate, 99999999))
 
     # 处理新出现的股票
-    codepre_list = get_codepre_list(connect, market, quotations)
-
     today = datetime.date.today()
     today = today.year * 10000 + today.month * 100 + today.day
     count = 0
@@ -274,10 +285,10 @@ def import_one_stock_data(
             today.year * 10000 + today.month * 100 + today.day
         ) * 10000 + 1500
     else:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     if today_datetime <= last_datetime:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     get_bars = (
         api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
@@ -312,27 +323,27 @@ def import_one_stock_data(
                 if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord[1]}, bar: {bar['open']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[2] - bar["high"]) / last_krecord[2] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord[2]}, bar: {bar['high']}")
-                    return 0
-                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01:
+                    return (0, False,  Datetime(last_datetime))
+                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01 and abs(last_krecord[3] - bar["low"]) > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord[3]}, bar: {bar['low']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[4] - bar["close"]) / last_krecord[4] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord[4]}, bar: {bar['close']}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]*0.001) / last_krecord[5] > 0.1:
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]) > 10000:
                     hku_error(
-                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[6] != 0.0 and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']}")
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[6] != 0.0 and last_krecord[6] != 0.0 and abs(last_krecord[6] - bar["vol"]) > 10000:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 continue
 
             if (
@@ -365,7 +376,7 @@ def import_one_stock_data(
                                            data=buf)
         connect.insert(context=ic, settings={"prefer_warmed_unmerged_parts_seconds": 86400})
 
-    return len(buf)
+    return (len(buf), True, Datetime(last_datetime))
 
 
 def update_stock_info(connect, market):
@@ -409,7 +420,7 @@ def clear_extern_data(connect, market, data_type):
         index_list = ('min15', 'min30', 'min60', 'hour2')
         lastdate = connect.command(
             f"select toInt32(max(date)) from hku_data.min5_k where market='SH' and code='000001'")
-        lastdate = Datetime.from_timestamp_utc(lastdate*1000000).start_of_day()
+        lastdate = Datetime.from_timestamp_utc(int(lastdate*1000000)).start_of_day()
         last_timestamp = Datetime(lastdate).timestamp_utc()//1000000
         for index_type in index_list:
             sql = f"delete from hku_data.{index_type}_k where market='{market}' and date>={last_timestamp}"
@@ -420,6 +431,16 @@ def clear_extern_data(connect, market, data_type):
         hku_info(f"清理 {market} {data_type} 线扩展数据完毕")
 
 
+@hku_catch(ret=None)
+def get_clickhouse_importer():
+    filename = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini'
+    config = ConfigParser()
+    config.read(filename, encoding='utf-8')
+    importer = KDataToClickHouseImporter()
+    return importer if importer.set_config(config.get("kdata", "host"), config.getint("kdata", "port", fallback=9000),
+                                           config.get("kdata", "usr"), config.get("kdata", "pwd")) else None
+
+
 @hku_catch(trace=True, re_raise=True)
 def import_data(
     connect,
@@ -460,6 +481,9 @@ def import_data(
             for index_type in index_list:
                 update_data[index_type] = []
 
+    failed_limit = 20
+    failed_count = 0
+    failed_list = []
     total = len(stock_list)
     # market, code, valid, type
     for i, stock in enumerate(stock_list):
@@ -468,9 +492,15 @@ def import_data(
                 progress(i, total)
             continue
 
-        this_count = import_one_stock_data(
+        this_count, success, lastdate = import_one_stock_data(
             connect, api, market, ktype, stock, startDate
         )
+        if not success:
+            failed_count += 1
+            failed_list.append((market, stock[1], lastdate))
+        if failed_count >= failed_limit:
+            # hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+            break
         add_record_count += this_count
         if ktype in ("DAY", "5MIN"):
             if ktype == "DAY":
@@ -490,6 +520,9 @@ def import_data(
         if progress:
             progress(i, total)
 
+    if total > 0 and progress:
+        progress(total, total)
+
     if ktype in ("DAY", "5MIN"):
         for index_type in index_list:
             if len(update_data[index_type]) > 0:
@@ -501,6 +534,25 @@ def import_data(
                 update_data[index_type].clear()
         update_data.clear()
 
+    if 0 < failed_count < failed_limit and is_valid_license():
+        # 删除最后记录
+        ktype_dict = {
+            'DAY': 'DAY',
+            '1MIN': 'MIN',
+            '5MIN': 'MIN5'
+        }
+        nktype = ktype_dict[ktype]
+        ch_importer = get_clickhouse_importer()
+        if ch_importer is not None:
+            for r in failed_list:
+                hku_warn(f"清理 {r[0]} {r[1]} {nktype} {r[2].start_of_day()}")
+                ch_importer.remove(r[0], r[1], nktype, r[2].start_of_day())
+            hku_warn(f"已清理 {market} {failed_count} 个失败股票的最后记录，建议重新导入")
+
+    if failed_count >= failed_limit:
+        hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+        return add_record_count
+
     cur_year = Datetime.today().year
     if ktype == "DAY":
         update_stock_info(connect, market)
@@ -592,15 +644,16 @@ def import_on_stock_trans(connect, api, market, stock_record, max_days):
                 if second > 59:
                     continue
 
-                trans_buf.append(
-                    (
-                        market, code,
-                        Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
-                        int(roundEx(record["price"], 3) * 1000.0),
-                        int(record["vol"]),
-                        record["buyorsell"],
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    trans_buf.append(
+                        (
+                            market, code,
+                            Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
+                            int(roundEx(record["price"], 3) * 1000.0),
+                            int(record["vol"]),
+                            record["buyorsell"],
+                        )
                     )
-                )
             except Exception as e:
                 hku_error("Failed trans to record! {}", e)
 
@@ -701,8 +754,9 @@ def import_on_stock_time(connect, api, market, stock_record, max_days):
             elif time == 1360:
                 time = 1400
             try:
-                time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
-                                ticks, int(roundEx(record['price'], 3) * 1000.0), int(record['vol'])))
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
+                                     ticks, int(roundEx(record['price'], 3) * 1000.0), int(record['vol'])))
                 time += 1
             except Exception as e:
                 hku_error("Failed trans record {}! {}".format(record, e))

hikyuu/data/pytdx_to_h5.py

@@ -23,13 +23,15 @@
 # SOFTWARE.
 
 import sys
+import os
 import math
 import datetime
 import sqlite3
 from pytdx.hq import TDXParams
+from configparser import ConfigParser
 
 from hikyuu.util.mylog import get_default_logger, hku_error, hku_debug
-from hikyuu import Datetime
+from hikyuu import Datetime, is_valid_license, KDataToHdf5Importer
 from hikyuu.data.common import *
 from hikyuu.data.common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from hikyuu.data.common_sqlite3 import (
@@ -235,6 +237,7 @@ def guess_5min_n_step(last_datetime):
     return (n, step)
 
 
+@hku_catch(trace=True)
 def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, startDate=199012191500):
     market = market.upper()
     pytdx_market = to_pytdx_market(market)
@@ -246,9 +249,9 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
     table = get_h5table(h5file, market, code)
     if table is None:
         hku_error("Can't get table({}{})".format(market, code))
-        return 0
+        return (0, True, Datetime())
 
-    last_datetime = table[-1]['datetime'] if table.nrows > 0 else startDate
+    last_datetime = int(table[-1]['datetime']) if table.nrows > 0 else startDate
 
     today = datetime.date.today()
     if ktype == 'DAY':
@@ -266,10 +269,10 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
         pytdx_kline_type = TDXParams.KLINE_TYPE_5MIN
         today_datetime = (today.year * 10000 + today.month * 100 + today.day) * 10000 + 1500
     else:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     if today_datetime <= last_datetime:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     get_bars = api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
 
@@ -311,27 +314,27 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
                 if abs(last_krecord['openPrice']*0.001 - bar["open"]) / (last_krecord['openPrice']*0.001) > 0.02:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord['openPrice']*0.001}, bar: {bar['open']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord['highPrice']*0.001 - bar["high"]) / (last_krecord['highPrice']*0.001) > 0.02:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord['highPrice']*0.001}, bar: {bar['high']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord['lowPrice']*0.001 - bar["low"]) / (last_krecord['lowPrice']*0.001) > 0.02:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord['lowPrice']*0.001}, bar: {bar['low']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord['closePrice']*0.001 - bar["close"]) / (last_krecord['closePrice']*0.001) > 0.02:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord['closePrice']*0.001}, bar: {bar['close']}")
-                    return 0
-                if ktype == 'DAY' and last_krecord['transAmount'] != 0.0 and abs(last_krecord['transAmount'] - round(bar["amount"]*0.001)) / last_krecord['transAmount'] > 0.1:
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord['transAmount'] != 0 and (abs(float(last_krecord['transAmount']) - round(bar["amount"]*0.001))) > 10:
                     hku_error(
-                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord['transAmount']}, bar: {bar['amount']*0.001}")
-                    return 0
-                if ktype == 'DAY' and last_krecord['transCount'] != 0.0 and abs(last_krecord['transCount'] - round(bar["vol"])) / last_krecord['transCount'] > 0.1:
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {float(last_krecord['transAmount'])}, bar: {round(bar['amount']*0.001)}")
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord['transCount'] != 0 and abs(float(last_krecord['transCount']) - bar["vol"]) > 10:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord['transCount']}, bar: {bar['vol']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 continue
 
             if today_datetime >= bar_datetime > last_datetime \
@@ -378,7 +381,16 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
         table.remove()
 
     # table.close()
-    return add_record_count
+    return (add_record_count, True, Datetime(last_datetime))
+
+
+@hku_catch(ret=None)
+def get_hdf5_importer(market, ktype):
+    filename = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini'
+    config = ConfigParser()
+    config.read(filename, encoding='utf-8')
+    importer = KDataToHdf5Importer()
+    return importer if importer.set_config(config.get("hikyuu", "datadir"), [market], [ktype]) else None
 
 
 def import_data(connect, market, ktype, quotations, api, dest_dir, startDate=199012190000, progress=ProgressBar):
@@ -401,6 +413,9 @@ def import_data(connect, market, ktype, quotations, api, dest_dir, startDate=199
     if not stock_list:
         return 0
 
+    failed_limit = 20
+    failed_count = 0
+    failed_list = []
     total = len(stock_list)
     for i, stock in enumerate(stock_list):
         if stock[3] == 0 or len(stock[2]) != 6:
@@ -408,7 +423,12 @@ def import_data(connect, market, ktype, quotations, api, dest_dir, startDate=199
                 progress(i, total)
             continue
 
-        this_count = import_one_stock_data(connect, api, h5file, market, ktype, stock, startDate)
+        this_count, success, lastdate = import_one_stock_data(connect, api, h5file, market, ktype, stock, startDate)
+        if not success:
+            failed_count += 1
+            failed_list.append((market, stock[2], lastdate))
+        if failed_count >= failed_limit:
+            break
         add_record_count += this_count
         if this_count > 0:
             if ktype == 'DAY':
@@ -418,8 +438,31 @@ def import_data(connect, market, ktype, quotations, api, dest_dir, startDate=199
         if progress:
             progress(i, total)
 
+    if total > 0 and progress:
+        progress(total, total)
+
     connect.commit()
     h5file.close()
+
+    if 0 < failed_count < failed_limit and is_valid_license():
+        # 删除最后记录
+        ktype_dict = {
+            'DAY': 'DAY',
+            '1MIN': 'MIN',
+            '5MIN': 'MIN5'
+        }
+        nktype = ktype_dict[ktype]
+        h5_importer = get_hdf5_importer(market, nktype)
+        if h5_importer is not None:
+            for r in failed_list:
+                hku_warn("清理 {}{} {}: {}", r[0], r[1], nktype, r[2].start_of_day())
+                h5_importer.remove(r[0], r[1], nktype, r[2].start_of_day())
+            hku_warn(f"已清理 {market} {failed_count} 个失败股票的最后记录，建议重新导入")
+
+    if failed_count >= failed_limit:
+        hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+        return add_record_count
+
     return add_record_count
 
 
@@ -479,12 +522,13 @@ def import_on_stock_trans(connect, api, h5file, market, stock_record, max_days):
                     second += 3
                 if second > 59:
                     continue
-                row['datetime'] = cur_date * 1000000 + minute * 100 + second
-                row['price'] = int(record['price'] * 1000)
-                row['vol'] = record['vol']
-                row['buyorsell'] = record['buyorsell']
-                row.append()
-                add_record_count += 1
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    row['datetime'] = cur_date * 1000000 + minute * 100 + second
+                    row['price'] = int(record['price'] * 1000)
+                    row['vol'] = record['vol']
+                    row['buyorsell'] = record['buyorsell']
+                    row.append()
+                    add_record_count += 1
             except Exception as e:
                 hku_error("Failed trans to record! {}", e)
 
@@ -578,12 +622,13 @@ def import_on_stock_time(connect, api, h5file, market, stock_record, max_days):
             elif time == 1360:
                 time = 1400
             try:
-                row['datetime'] = this_date + time
-                row['price'] = int(record['price'] * 1000)
-                row['vol'] = record['vol']
-                row.append()
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    row['datetime'] = this_date + time
+                    row['price'] = int(record['price'] * 1000)
+                    row['vol'] = record['vol']
+                    row.append()
+                    add_record_count += 1
                 time += 1
-                add_record_count += 1
             except Exception as e:
                 hku_error("Failed trans record {}! {}".format(record, e))
 
@@ -624,7 +669,7 @@ if __name__ == '__main__':
     import time
     starttime = time.time()
 
-    dest_dir = "d:\\stock"
+    dest_dir = "/Users/fasiondog/stock"
     tdx_server = '180.101.48.170'
     tdx_port = 7709
     quotations = ['stock', 'fund']

hikyuu/data/pytdx_to_mysql.py

@@ -23,15 +23,17 @@
 # SOFTWARE.
 
 import sys
+import os
 import math
 import datetime
 from pytdx.hq import TDXParams
+from configparser import ConfigParser
 
 from hikyuu.util.mylog import hku_error, hku_debug
 
 import mysql.connector
 
-from hikyuu import Datetime
+from hikyuu import Datetime, is_valid_license, KDataToMySQLImporter
 from .common import *
 from .common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from .common_mysql import (
@@ -318,10 +320,10 @@ def import_one_stock_data(
             today.year * 10000 + today.month * 100 + today.day
         ) * 10000 + 1500
     else:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     if today_datetime <= last_datetime:
-        return 0
+        return (0, True, Datetime(last_datetime))
 
     get_bars = (
         api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
@@ -356,27 +358,27 @@ def import_one_stock_data(
                 if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord[1]}, bar: {bar['open']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[2] - bar["high"]) / last_krecord[2] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord[2]}, bar: {bar['high']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord[3]}, bar: {bar['low']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 if abs(last_krecord[4] - bar["close"]) / last_krecord[4] > 0.01:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord[4]}, bar: {bar['close']}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]*0.001) / last_krecord[5] > 0.1:
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]*0.001) > 10:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
-                    return 0
-                if ktype == 'DAY' and last_krecord[6] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                    return (0, False, Datetime(last_datetime))
+                if ktype == 'DAY' and last_krecord[6] != 0.0 and abs(last_krecord[6] - bar["vol"]) > 10:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
-                    return 0
+                    return (0, False, Datetime(last_datetime))
                 continue
 
             if (
@@ -440,7 +442,17 @@ def import_one_stock_data(
         connect.commit()
         cur.close()
 
-    return len(buf)
+    return (len(buf), True, Datetime(last_datetime))
+
+
+@hku_catch(ret=None)
+def get_mysql_importer():
+    filename = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini'
+    config = ConfigParser()
+    config.read(filename, encoding='utf-8')
+    importer = KDataToMySQLImporter()
+    return importer if importer.set_config(config.get("kdata", "host"), config.getint("kdata", "port", fallback=3306),
+                                           config.get("kdata", "usr"), config.get("kdata", "pwd")) else None
 
 
 @hku_catch(trace=True, re_raise=True)
@@ -470,6 +482,9 @@ def import_data(
 
     stock_list = get_stock_list(connect, market, quotations)
 
+    failed_limit = 20
+    failed_count = 0
+    failed_list = []
     total = len(stock_list)
     for i, stock in enumerate(stock_list):
         if stock[3] == 0 or len(stock[2]) != 6:
@@ -477,9 +492,14 @@ def import_data(
                 progress(i, total)
             continue
 
-        this_count = import_one_stock_data(
+        this_count, success, lastdate = import_one_stock_data(
             connect, api, market, ktype, stock, startDate
         )
+        if not success:
+            failed_count += 1
+            failed_list.append((market, stock[2], lastdate))
+        if failed_count >= failed_limit:
+            break
         add_record_count += this_count
         if this_count > 0:
             if ktype == "DAY":
@@ -491,6 +511,28 @@ def import_data(
             progress(i, total)
 
     connect.commit()
+
+    if total > 0 and progress:
+        progress(total, total)
+
+    if 0 < failed_count < failed_limit and is_valid_license():
+        # 删除最后记录
+        ktype_dict = {
+            'DAY': 'DAY',
+            '1MIN': 'MIN',
+            '5MIN': 'MIN5'
+        }
+        nktype = ktype_dict[ktype]
+        h5_importer = get_mysql_importer(market, nktype)
+        if h5_importer is not None:
+            for r in failed_list:
+                hku_warn("remove {}{} {}: {}", r[0], r[1], nktype, r[2].start_of_day())
+                h5_importer.remove(r[0], r[1], nktype, r[2].start_of_day())
+            hku_warn(f"已清理 {market} {failed_count} 个失败股票的最后记录，建议重新导入")
+
+    if failed_count >= failed_limit:
+        hku_error(f"{market} {ktype} 连续失败20个股票，已停止导入, 建议重新导入")
+        return add_record_count
     return add_record_count
 
 
@@ -588,14 +630,15 @@ def import_on_stock_trans(connect, api, market, stock_record, max_days):
                 if second > 59:
                     continue
 
-                trans_buf.append(
-                    (
-                        cur_date * 1000000 + minute * 100 + second,
-                        record["price"],
-                        record["vol"],
-                        record["buyorsell"],
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    trans_buf.append(
+                        (
+                            cur_date * 1000000 + minute * 100 + second,
+                            record["price"],
+                            record["vol"],
+                            record["buyorsell"],
+                        )
                     )
-                )
             except Exception as e:
                 hku_error("Failed trans to record! {}", e)
 
@@ -721,7 +764,8 @@ def import_on_stock_time(connect, api, market, stock_record, max_days):
             elif time == 1360:
                 time = 1400
             try:
-                time_buf.append((this_date + time, record['price'], record['vol']))
+                if record['price'] > 0.0 and record['vol'] >= 0.0:
+                    time_buf.append((this_date + time, record['price'], record['vol']))
                 time += 1
             except Exception as e:
                 hku_error("Failed trans record {}! {}".format(record, e))

hikyuu/data/pytdx_weight_to_clickhouse.py

@@ -55,6 +55,8 @@ def pytdx_import_weight_to_clickhouse(pytdx_api, connect, market):
             db_last_date, last_total_count, last_free_count = (0, 0, 0)
 
         xdxr_list = pytdx_api.get_xdxr_info(pytdx_market, code)
+        if xdxr_list is None:
+            continue
         update_last_db_weight = False
         new_last_db_weight = last_db_weight
         records = {}

hikyuu/data/pytdx_weight_to_mysql.py

@@ -67,6 +67,8 @@ def pytdx_import_weight_to_mysql(pytdx_api, connect, market):
         cur.close()
 
         xdxr_list = pytdx_api.get_xdxr_info(pytdx_market, code)
+        if xdxr_list is None:
+            continue
         update_last_db_weight = False
         new_last_db_weight = last_db_weight
         records = {}

hikyuu/data/pytdx_weight_to_sqlite.py

@@ -64,6 +64,8 @@ def pytdx_import_weight_to_sqlite(pytdx_api, connect, market):
         cur.close()
 
         xdxr_list = pytdx_api.get_xdxr_info(pytdx_market, code)
+        if xdxr_list is None:
+            continue
         update_last_db_weight = False
         new_last_db_weight = last_db_weight
         records = {}

hikyuu/data/sqlite_upgrade/0029.sql

@@ -0,0 +1,4 @@
+BEGIN TRANSACTION;
+update `stocktypeinfo` set `minTradeNumber`=100 where `type`=5 and `description`='ETF';
+UPDATE `version` set `version` = 29;
+COMMIT;