hikyuu 2.6.3__py3-none-win_amd64.whl → 2.6.6__py3-none-win_amd64.whl

hikyuu/data/pytdx_to_h5.py

@@ -29,7 +29,7 @@ import sqlite3
 from pytdx.hq import TDXParams
 
 from hikyuu.util.mylog import get_default_logger, hku_error, hku_debug
-
+from hikyuu import Datetime
 from hikyuu.data.common import *
 from hikyuu.data.common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from hikyuu.data.common_sqlite3 import (
@@ -191,7 +191,10 @@ def guess_day_n_step(last_datetime):
     if n < 1:
         last_m = last_date // 100 - last_y * 100
         last_d = last_date - (last_y * 10000 + last_m * 100)
-        step = (today - datetime.date(last_y, last_m, last_d)).days
+        step = (today - datetime.date(last_y, last_m, last_d)).days + 1
+        if step > 800:
+            n = 1
+            step = 800
 
     return (n, step)
 
@@ -204,10 +207,10 @@ def guess_1min_n_step(last_datetime):
     last_m = last_date // 100 - last_y * 100
     last_d = last_date - (last_y * 10000 + last_m * 100)
 
-    n = int((today - datetime.date(last_y, last_m, last_d)).days * 240 // 800)
+    n = int(((today - datetime.date(last_y, last_m, last_d)).days * 240 + 1) // 800)
     step = 800
     if n < 1:
-        step = (today - datetime.date(last_y, last_m, last_d)).days * 240
+        step = (today - datetime.date(last_y, last_m, last_d)).days * 240 + 1
     elif n > 99:
         n = 99
 
@@ -222,10 +225,10 @@ def guess_5min_n_step(last_datetime):
     last_m = last_date // 100 - last_y * 100
     last_d = last_date - (last_y * 10000 + last_m * 100)
 
-    n = int((today - datetime.date(last_y, last_m, last_d)).days * 48 // 800)
+    n = int(((today - datetime.date(last_y, last_m, last_d)).days * 48 + 1) // 800)
     step = 800
     if n < 1:
-        step = (today - datetime.date(last_y, last_m, last_d)).days * 48
+        step = (today - datetime.date(last_y, last_m, last_d)).days * 48 + 1
     elif n > 99:
         n = 99
 
@@ -270,6 +273,12 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
 
     get_bars = api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
 
+    need_check = False
+    last_krecord = None
+    if table.nrows > 0:
+        need_check = True
+        last_krecord = table[-1]
+
     add_record_count = 0
 
     row = table.row
@@ -277,7 +286,7 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
         bar_list = get_bars(pytdx_kline_type, pytdx_market, code, n * 800, step)
         n -= 1
         if bar_list is None:
-            # print(code, "invalid!!")
+            print(code, "invalid!!")
             continue
 
         for bar in bar_list:
@@ -293,6 +302,34 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
                 hku_error("Failed translate datetime: {}, from {}! {}".format(bar, api.ip, e))
                 continue
 
+            if need_check and bar_datetime == last_datetime:
+                # print(f'{bar_datetime} {ktype} {market}{code}')
+                if abs(last_krecord['openPrice']*0.001 - bar["open"]) / (last_krecord['openPrice']*0.001) > 0.02:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord['openPrice']*0.001}, bar: {bar['open']}")
+                    return 0
+                if abs(last_krecord['highPrice']*0.001 - bar["high"]) / (last_krecord['highPrice']*0.001) > 0.02:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord['highPrice']*0.001}, bar: {bar['high']}")
+                    return 0
+                if abs(last_krecord['lowPrice']*0.001 - bar["low"]) / (last_krecord['lowPrice']*0.001) > 0.02:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord['lowPrice']*0.001}, bar: {bar['low']}")
+                    return 0
+                if abs(last_krecord['closePrice']*0.001 - bar["close"]) / (last_krecord['closePrice']*0.001) > 0.02:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord['closePrice']*0.001}, bar: {bar['close']}")
+                    return 0
+                if ktype == 'DAY' and last_krecord['transAmount'] != 0.0 and abs(last_krecord['transAmount'] - round(bar["amount"]*0.001)) / last_krecord['transAmount'] > 0.1:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord['transAmount']}, bar: {bar['amount']*0.001}")
+                    return 0
+                if ktype == 'DAY' and last_krecord['transAmount'] != 0.0 and abs(last_krecord['transCount'] - round(bar["vol"])) / last_krecord['transCount'] > 0.1:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord['transCount']}, bar: {bar['vol']}")
+                    return 0
+                continue
+
             if today_datetime >= bar_datetime > last_datetime \
                     and bar['high'] >= bar['open'] >= bar['low'] > 0 \
                     and bar['high'] >= bar['close'] >= bar['low'] > 0 \
@@ -584,7 +621,7 @@ if __name__ == '__main__':
     starttime = time.time()
 
     dest_dir = "d:\\stock"
-    tdx_server = '119.147.212.81'
+    tdx_server = '180.101.48.170'
     tdx_port = 7709
     quotations = ['stock', 'fund']
 
@@ -601,10 +638,11 @@ if __name__ == '__main__':
     add_count = import_index_name(connect)
     print("指数个数：", add_count)
     """
-    print("导入股票代码表")
-    add_count = import_stock_name(connect, api, 'SH', quotations)
-    # add_count += import_stock_name(connect, api, 'SZ', quotations)
-    print("新增股票数：", add_count)
+    # print("导入股票代码表")
+    # add_count = import_stock_name(connect, api, 'SH', quotations)
+    # # add_count += import_stock_name(connect, api, 'SZ', quotations)
+    # print("新增股票数：", add_count)
+
     """
     print("\n导入上证日线数据")
     add_count = import_data(connect, 'SH', 'DAY', quotations, api, dest_dir, progress=ProgressBar)
@@ -613,11 +651,13 @@ if __name__ == '__main__':
     print("\n导入深证日线数据")
     add_count = import_data(connect, 'SZ', 'DAY', quotations, api, dest_dir, progress=ProgressBar)
     print("\n导入数量：", add_count)
+    """
 
     print("\n导入上证5分钟线数据")
-    add_count = import_data(connect, 'SH', '5MIN', quotations, api, dest_dir, progress=ProgressBar)
+    add_count = import_data(connect, 'SH', 'DAY', quotations, api, dest_dir, progress=ProgressBar)
     print("\n导入数量：", add_count)
 
+    """
     print("\n导入深证5分钟线数据")
     add_count = import_data(connect, 'SZ', '5MIN', quotations, api, dest_dir, progress=ProgressBar)
     print("\n导入数量：", add_count)

hikyuu/data/pytdx_to_mysql.py

@@ -31,6 +31,7 @@ from hikyuu.util.mylog import hku_error, hku_debug
 
 import mysql.connector
 
+from hikyuu import Datetime
 from .common import *
 from .common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from .common_mysql import (
@@ -40,6 +41,7 @@ from .common_mysql import (
     get_stock_list,
     get_table,
     get_lastdatetime,
+    get_last_krecord,
     update_extern_data,
 )
 from .weight_to_mysql import qianlong_import_weight
@@ -238,8 +240,10 @@ def guess_day_n_step(last_datetime):
     if n < 1:
         last_m = last_date // 100 - last_y * 100
         last_d = last_date - (last_y * 10000 + last_m * 100)
-        step = (today - datetime.date(last_y, last_m, last_d)).days
-
+        step = (today - datetime.date(last_y, last_m, last_d)).days + 1
+        if step > 800:
+            n = 1
+            step = 800
     return (n, step)
 
 
@@ -251,10 +255,10 @@ def guess_1min_n_step(last_datetime):
     last_m = last_date // 100 - last_y * 100
     last_d = last_date - (last_y * 10000 + last_m * 100)
 
-    n = int((today - datetime.date(last_y, last_m, last_d)).days * 240 // 800)
+    n = int(((today - datetime.date(last_y, last_m, last_d)).days * 240 + 1) // 800)
     step = 800
     if n < 1:
-        step = (today - datetime.date(last_y, last_m, last_d)).days * 240
+        step = (today - datetime.date(last_y, last_m, last_d)).days * 240 + 1
     elif n > 99:
         n = 99
 
@@ -269,10 +273,10 @@ def guess_5min_n_step(last_datetime):
     last_m = last_date // 100 - last_y * 100
     last_d = last_date - (last_y * 10000 + last_m * 100)
 
-    n = int((today - datetime.date(last_y, last_m, last_d)).days * 48 // 800)
+    n = int(((today - datetime.date(last_y, last_m, last_d)).days * 48 + 1) // 800)
     step = 800
     if n < 1:
-        step = (today - datetime.date(last_y, last_m, last_d)).days * 48
+        step = (today - datetime.date(last_y, last_m, last_d)).days * 48 + 1
     elif n > 99:
         n = 99
 
@@ -288,9 +292,11 @@ def import_one_stock_data(
     stockid, marketid, code, valid, stktype = stock_record[:5]
     hku_debug("{}{}".format(market, code))
     table = get_table(connect, market, code, ktype)
-    last_datetime = get_lastdatetime(connect, table)
-    if last_datetime is None:
-        last_datetime = startDate
+    last_krecord = get_last_krecord(connect, table)
+    last_datetime = startDate if last_krecord is None else last_krecord[0]
+    # last_datetime = get_lastdatetime(connect, table)
+    # if last_datetime is None:
+    #     last_datetime = startDate
 
     today = datetime.date.today()
     if ktype == "DAY":
@@ -342,6 +348,33 @@ def import_one_stock_data(
                 hku_error("Failed translate datetime: {}, from {}! {}".format(bar, api.ip, e))
                 continue
 
+            if last_krecord is not None and bar_datetime == last_datetime:
+                if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord open: {last_krecord[1]}, bar: {bar['open']}")
+                    return 0
+                if abs(last_krecord[2] - bar["high"]) / last_krecord[2] > 0.01:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord high: {last_krecord[2]}, bar: {bar['high']}")
+                    return 0
+                if abs(last_krecord[3] - bar["low"]) / last_krecord[3] > 0.01:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord low: {last_krecord[3]}, bar: {bar['low']}")
+                    return 0
+                if abs(last_krecord[4] - bar["close"]) / last_krecord[4] > 0.01:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord close: {last_krecord[4]}, bar: {bar['close']}")
+                    return 0
+                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[5] - bar["amount"]*0.001) / last_krecord[5] > 0.1:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
+                    return 0
+                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                    hku_error(
+                        f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
+                    return 0
+                continue
+
             if (
                 today_datetime >= bar_datetime > last_datetime
                 and bar["high"] >= bar["open"] >= bar["low"] > 0