hikyuu 2.6.8.5__py3-none-win_amd64.whl → 2.6.9__py3-none-win_amd64.whl

hikyuu/data/em_block_to_clickhouse.py

@@ -4,85 +4,32 @@
 # Create on: 20240102
 #    Author: fasiondog
 
-from concurrent.futures import ThreadPoolExecutor
-from hikyuu.data.common import MARKET, get_stk_code_name_list
 from hikyuu.util import *
 from hikyuu.fetcher.stock.zh_block_em import *
+from hikyuu.data.download_block import *
 
 
-def em_import_block_to_clickhouse(connect, code_market_dict, categorys=('行业板块', '指数板块')):
-    all_block_info = {}
-
-    with ThreadPoolExecutor(4) as executor:
-        if '行业板块' in categorys:
-            t1 = executor.submit(get_all_hybk_info, code_market_dict)
-
-        if '概念板块' in categorys:
-            t2 = executor.submit(get_all_gnbk_info, code_market_dict)
-
-        if '地域板块' in categorys:
-            t3 = executor.submit(get_all_dybk_info, code_market_dict)
-
-        if '指数板块' in categorys:
-            t4 = executor.submit(get_all_zsbk_info, code_market_dict)
-
-        success_fetch_hy = False
-        if '行业板块' in categorys:
-            x = t1.result()
-            hku_info("获取行业板块信息完毕")
-            if x:
-                all_block_info["行业板块"] = x
-                success_fetch_hy = True
-
-        success_fetch_gn = False
-        if '概念板块' in categorys:
-            x = t2.result()
-            hku_info("获取概念板块信息完毕")
-            if x:
-                all_block_info["概念板块"] = x
-                success_fetch_gn = True
-
-        success_fetch_dy = False
-        if '地域板块' in categorys:
-            x = t3.result()
-            hku_info("获取地域板块信息完毕")
-            if x:
-                all_block_info["地域板块"] = x
-                success_fetch_dy = True
-
-        success_fetch_zs = False
-        if '指数板块' in categorys:
-            x = t4.result()
-            hku_info("获取指数板块信息完毕")
-            if x:
-                all_block_info["指数板块"] = x
-                success_fetch_zs = True
-
-    blks = []
-    if success_fetch_hy:
-        blks.append('行业板块')
-    if success_fetch_gn:
-        blks.append('概念板块')
-    if success_fetch_dy:
-        blks.append('地域板块')
-    if success_fetch_zs:
-        blks.append('指数板块')
+def em_import_block_to_clickhouse(connect):
+    download_block_info()
+
+    blocks_info = read_block_from_path()
+    blks = blocks_info['block']
+    blks_info = blocks_info['block_info']
 
     if not blks:
-        return
+        return 0
 
     hku_info("更新数据库")
-    if len(blks) == 1:
-        sql = f"delete from hku_base.block where category in ('{blks[0]}')"
-    else:
-        sql = f"delete from hku_base.block where category in {tuple(blks)}"
+    sql = f"delete from hku_base.block where category in {tuple(blks.keys())}"
+    connect.command(sql)
+    sql = f"delete from hku_base.blockindex where category in {tuple(blks.keys())}"
     connect.command(sql)
 
     insert_records = []
 
-    for category in all_block_info:
-        for name in all_block_info[category]:
-            for code in all_block_info[category][name]:
+    for category, blocks in blks.items():
+        for name, codes in blocks.items():
+            for code in codes:
                 insert_records.append((category, name, code))
 
     if insert_records:
@@ -91,6 +38,17 @@ def em_import_block_to_clickhouse(connect, code_market_dict, categorys=('行业
                                            data=insert_records)
         connect.insert(context=ic)
 
+    index_records = []
+    for category, blocks in blks_info.items():
+        for name, index_code in blocks.items():
+            if len(index_code) == 8:
+                index_records.append((category, name, index_code))
+    if index_records:
+        ic = connect.create_insert_context(table='blockindex', database='hku_base',
+                                           column_names=['category', 'name', 'index_code'],
+                                           data=index_records)
+        connect.insert(context=ic)
+
     return len(insert_records)
 
 
@@ -109,12 +67,6 @@ if __name__ == "__main__":
     client = clickhouse_connect.get_client(
         host=host, username=user, password=password)
 
-    x = get_stk_code_name_list(MARKET.SH)
-    code_market_dict = {}
-    for v in x:
-        code_market_dict[v["code"]] = MARKET.SH
-    # print(code_market_dict)
-
-    em_import_block_to_clickhouse(client, code_market_dict, categorys=('行业板块', ))
+    em_import_block_to_clickhouse(client)
 
     client.close()

hikyuu/data/em_block_to_mysql.py

@@ -4,93 +4,49 @@
 # Create on: 20240102
 #    Author: fasiondog
 
-from concurrent.futures import ThreadPoolExecutor
-from hikyuu.data.common import MARKET, get_stk_code_name_list
+
 from hikyuu.util import *
 from hikyuu.fetcher.stock.zh_block_em import *
+from hikyuu.data.download_block import *
+
 
+def em_import_block_to_mysql(connect):
+    download_block_info()
 
-def em_import_block_to_mysql(connect, code_market_dict, categorys=('行业板块', '概念板块', '地域板块', '指数板块')):
-    all_block_info = {}
-
-    with ThreadPoolExecutor(4) as executor:
-        if '行业板块' in categorys:
-            t1 = executor.submit(get_all_hybk_info, code_market_dict)
-
-        if '概念板块' in categorys:
-            t2 = executor.submit(get_all_gnbk_info, code_market_dict)
-
-        if '地域板块' in categorys:
-            t3 = executor.submit(get_all_dybk_info, code_market_dict)
-
-        if '指数板块' in categorys:
-            t4 = executor.submit(get_all_zsbk_info, code_market_dict)
-
-        success_fetch_hy = False
-        if '行业板块' in categorys:
-            x = t1.result()
-            hku_info("获取行业板块信息完毕")
-            if x:
-                all_block_info["行业板块"] = x
-                success_fetch_hy = True
-
-        success_fetch_gn = False
-        if '概念板块' in categorys:
-            x = t2.result()
-            hku_info("获取概念板块信息完毕")
-            if x:
-                all_block_info["概念板块"] = x
-                success_fetch_gn = True
-
-        success_fetch_dy = False
-        if '地域板块' in categorys:
-            x = t3.result()
-            hku_info("获取地域板块信息完毕")
-            if x:
-                all_block_info["地域板块"] = x
-                success_fetch_dy = True
-
-        success_fetch_zs = False
-        if '指数板块' in categorys:
-            x = t4.result()
-            hku_info("获取指数板块信息完毕")
-            if x:
-                all_block_info["指数板块"] = x
-                success_fetch_zs = True
-
-    blks = []
-    if success_fetch_hy:
-        blks.append('行业板块')
-    if success_fetch_gn:
-        blks.append('概念板块')
-    if success_fetch_dy:
-        blks.append('地域板块')
-    if success_fetch_zs:
-        blks.append('指数板块')
+    blocks_info = read_block_from_path()
+    blks = blocks_info['block']
+    blks_info = blocks_info['block_info']
 
     if not blks:
-        return
+        return 0
 
     hku_info("更新数据库")
     cur = connect.cursor()
-    if len(blks) == 1:
-        sql = f"delete from hku_base.block where category in ('{blks[0]}')"
-    else:
-        sql = f"delete from hku_base.block where category in {tuple(blks)}"
+    sql = f"delete from hku_base.block where category in {tuple(blks.keys())}"
+    cur.execute(sql)
+    sql = f"delete from hku_base.BlockIndex where category in {tuple(blks.keys())}"
     cur.execute(sql)
 
     insert_records = []
 
-    for category in all_block_info:
-        for name in all_block_info[category]:
-            for code in all_block_info[category][name]:
+    for category, blocks in blks.items():
+        for name, codes in blocks.items():
+            for code in codes:
                 insert_records.append((category, name, code))
 
     if insert_records:
         sql = "insert into hku_base.block (category, name, market_code) values (%s,%s,%s)"
-        hku_info(f"insert block records: {len(insert_records)}")
         cur.executemany(sql, insert_records)
 
+    index_records = []
+    for category, blocks in blks_info.items():
+        for name, index_code in blocks.items():
+            if len(index_code) == 8:
+                index_records.append((category, name, index_code))
+    if index_records:
+        sql = "insert into hku_base.BlockIndex (category, name, market_code) values (%s,%s,%s)"
+        cur.executemany(sql, index_records)
+
     connect.commit()
     cur.close()
     return len(insert_records)
@@ -111,11 +67,5 @@ if __name__ == "__main__":
     connect = mysql.connector.connect(user=usr, password=pwd, host=host, port=port)
     create_database(connect)
 
-    x = get_stk_code_name_list(MARKET.SH)
-    code_market_dict = {}
-    for v in x:
-        code_market_dict[v["code"]] = MARKET.SH
-    # print(code_market_dict)
-
-    em_import_block_to_mysql(connect, code_market_dict)
+    em_import_block_to_mysql(connect)
     connect.close()

hikyuu/data/em_block_to_sqlite.py

@@ -4,97 +4,51 @@
 # Create on: 20240102
 #    Author: fasiondog
 
-from concurrent.futures import ThreadPoolExecutor
-from hikyuu.data.common import MARKET, get_stk_code_name_list
+
 from hikyuu.util import *
 from hikyuu.fetcher.stock.zh_block_em import *
+from hikyuu.data.download_block import *
 
 
 @spend_time
-def em_import_block_to_sqlite(connect, code_market_dict, categorys=('行业板块', '概念板块', '地域板块', '指数板块')):
-    all_block_info = {}
-
-    with ThreadPoolExecutor(4) as executor:
-        if '行业板块' in categorys:
-            t1 = executor.submit(get_all_hybk_info, code_market_dict)
-
-        if '概念板块' in categorys:
-            t2 = executor.submit(get_all_gnbk_info, code_market_dict)
-
-        if '地域板块' in categorys:
-            t3 = executor.submit(get_all_dybk_info, code_market_dict)
-
-        if '指数板块' in categorys:
-            t4 = executor.submit(get_all_zsbk_info, code_market_dict)
-
-        success_fetch_hy = False
-        if '行业板块' in categorys:
-            x = t1.result()
-            hku_info("获取行业板块信息完毕")
-            if x:
-                all_block_info["行业板块"] = x
-                success_fetch_hy = True
-
-        success_fetch_gn = False
-        if '概念板块' in categorys:
-            x = t2.result()
-            hku_info("获取概念板块信息完毕")
-            if x:
-                all_block_info["概念板块"] = x
-                success_fetch_gn = True
-
-        success_fetch_dy = False
-        if '地域板块' in categorys:
-            x = t3.result()
-            hku_info("获取地域板块信息完毕")
-            if x:
-                all_block_info["地域板块"] = x
-                success_fetch_dy = True
-
-        success_fetch_zs = False
-        if '指数板块' in categorys:
-            x = t4.result()
-            hku_info("获取指数板块信息完毕")
-            if x:
-                all_block_info["指数板块"] = x
-                success_fetch_zs = True
-
-    blks = []
-    if success_fetch_hy:
-        blks.append('行业板块')
-    if success_fetch_gn:
-        blks.append('概念板块')
-    if success_fetch_dy:
-        blks.append('地域板块')
-    if success_fetch_zs:
-        blks.append('指数板块')
-
-    print(blks)
+def em_import_block_to_sqlite(connect):
+    download_block_info()
+
+    blocks_info = read_block_from_path()
+    blks = blocks_info['block']
+    blks_info = blocks_info['block_info']
 
     if not blks:
-        return
+        return 0
 
     hku_info("更新数据库")
     cur = connect.cursor()
-    if len(blks) == 1:
-        sql = f"delete from block where category in ('{blks[0]}')"
-    else:
-        sql = f"delete from block where category in {tuple(blks)}"
-    hku_info(sql)
+    sql = f"delete from block where category in {tuple(blks.keys())}"
+    cur.execute(sql)
+    sql = f"delete from BlockIndex where category in {tuple(blks.keys())}"
     cur.execute(sql)
 
     insert_records = []
 
-    for category in all_block_info:
-        for name in all_block_info[category]:
-            for code in all_block_info[category][name]:
+    for category, blocks in blks.items():
+        for name, codes in blocks.items():
+            for code in codes:
                 insert_records.append((category, name, code))
 
     if insert_records:
         sql = "insert into block (category, name, market_code) values (?,?,?)"
-        hku_info(f"insert block records: {len(insert_records)}")
         cur.executemany(sql, insert_records)
 
+    index_records = []
+    for category, blocks in blks_info.items():
+        for name, index_code in blocks.items():
+            if len(index_code) == 8:
+                index_records.append((category, name, index_code))
+    if index_records:
+        sql = "insert into BlockIndex (category, name, market_code) values (?,?,?)"
+        cur.executemany(sql, index_records)
+        hku_info(f"insert block index records: {len(index_records)}")
+
     connect.commit()
     cur.close()
     return len(insert_records)
@@ -111,12 +65,6 @@ if __name__ == "__main__":
     connect = sqlite3.connect(dest_dir + "/stock.db")
     create_database(connect)
 
-    x = get_stk_code_name_list(MARKET.SH)
-    code_market_dict = {}
-    for v in x:
-        code_market_dict[v["code"]] = MARKET.SH
-    # print(code_market_dict)
-
-    em_import_block_to_sqlite(connect, code_market_dict, categorys=('地域板块',))
+    em_import_block_to_sqlite(connect)
 
     connect.close()

hikyuu/data/hku_config_template.py

@@ -210,8 +210,8 @@ future = False
 day = True
 min = True
 min5 = True
-trans = True
-time = True
+trans = False
+time = False
 day_start_date = 1990-12-19
 min_start_date = 2023-09-19
 min5_start_date = 2023-09-19
@@ -251,7 +251,7 @@ enable = False
 enable = False
 ;tmpdir = D:/stock
 ;host = 127.0.0.1
-:http_port = 8123
+;http_port = 8123
 ;port = 9000
 ;usr = root
 ;pwd = 

hikyuu/data/pytdx_to_clickhouse.py

@@ -29,7 +29,7 @@ from pytdx.hq import TDXParams
 
 from hikyuu.util import hku_error, hku_debug, hku_run_ignore_exception
 
-from hikyuu import Datetime
+from hikyuu import Datetime, roundEx
 from hikyuu.data.common import *
 from hikyuu.data.common_pytdx import to_pytdx_market, pytdx_get_day_trans
 from hikyuu.data.common_clickhouse import (
@@ -304,6 +304,10 @@ def import_one_stock_data(
                 hku_error("Failed translate datetime: {}, from {}! {}".format(bar, api.ip, e))
                 continue
 
+            if 'open' not in bar or 'high' not in bar or 'low' not in bar or 'close' not in bar or 'vol' not in bar or 'amount' not in bar:
+                hku_warn(f"Ignore invalid bar: {bar_datetime}, missing fields!")
+                continue
+
             if last_krecord is not None and bar_datetime == last_datetime:
                 if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
                     hku_error(
@@ -325,7 +329,7 @@ def import_one_stock_data(
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
                     return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                if ktype == 'DAY' and last_krecord[6] != 0.0 and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
                     return 0
@@ -343,12 +347,12 @@ def import_one_stock_data(
                         (
                             table[1], table[2],
                             Datetime(bar_datetime).timestamp_utc()//1000000,
-                            bar["open"],
-                            bar["high"],
-                            bar["low"],
-                            bar["close"],
-                            bar["amount"] * 0.001,
-                            bar["vol"]
+                            int(roundEx(bar["open"], 3)*1000),
+                            int(roundEx(bar["high"], 3)*1000),
+                            int(roundEx(bar["low"], 3)*1000),
+                            int(roundEx(bar["close"], 3)*1000),
+                            int(roundEx(bar["amount"], 0)),
+                            int(roundEx(bar["vol"], 3)*1000)
                             # bar['vol'] if stktype == 2 else round(bar['vol'] * 0.01)
                         )
                     )
@@ -592,8 +596,8 @@ def import_on_stock_trans(connect, api, market, stock_record, max_days):
                     (
                         market, code,
                         Datetime(cur_date * 1000000 + minute * 100 + second).timestamp_utc()//1000000,
-                        record["price"],
-                        record["vol"],
+                        int(roundEx(record["price"], 3) * 1000.0),
+                        int(record["vol"]),
                         record["buyorsell"],
                     )
                 )
@@ -698,7 +702,7 @@ def import_on_stock_time(connect, api, market, stock_record, max_days):
                 time = 1400
             try:
                 time_buf.append((market, code, Datetime(this_date + time).timestamp_utc() //
-                                ticks, record['price'], record['vol']))
+                                ticks, int(roundEx(record['price'], 3) * 1000.0), int(record['vol'])))
                 time += 1
             except Exception as e:
                 hku_error("Failed trans record {}! {}".format(record, e))

hikyuu/data/pytdx_to_h5.py

@@ -286,7 +286,7 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
         bar_list = get_bars(pytdx_kline_type, pytdx_market, code, n * 800, step)
         n -= 1
         if bar_list is None:
-            print(code, "invalid!!")
+            # print(code, "invalid!!")
             continue
 
         for bar in bar_list:
@@ -302,6 +302,10 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
                 hku_error("Failed translate datetime: {}, from {}! {}".format(bar, api.ip, e))
                 continue
 
+            if 'open' not in bar or 'high' not in bar or 'low' not in bar or 'close' not in bar or 'vol' not in bar or 'amount' not in bar:
+                hku_warn(f"Ignore invalid bar: {bar_datetime}, missing fields!")
+                continue
+
             if need_check and bar_datetime == last_datetime:
                 # print(f'{bar_datetime} {ktype} {market}{code}')
                 if abs(last_krecord['openPrice']*0.001 - bar["open"]) / (last_krecord['openPrice']*0.001) > 0.02:
@@ -324,7 +328,7 @@ def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, sta
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord['transAmount']}, bar: {bar['amount']*0.001}")
                     return 0
-                if ktype == 'DAY' and last_krecord['transAmount'] != 0.0 and abs(last_krecord['transCount'] - round(bar["vol"])) / last_krecord['transCount'] > 0.1:
+                if ktype == 'DAY' and last_krecord['transCount'] != 0.0 and abs(last_krecord['transCount'] - round(bar["vol"])) / last_krecord['transCount'] > 0.1:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord['transCount']}, bar: {bar['vol']}")
                     return 0

hikyuu/data/pytdx_to_mysql.py

@@ -348,6 +348,10 @@ def import_one_stock_data(
                 hku_error("Failed translate datetime: {}, from {}! {}".format(bar, api.ip, e))
                 continue
 
+            if 'open' not in bar or 'high' not in bar or 'low' not in bar or 'close' not in bar or 'vol' not in bar or 'amount' not in bar:
+                hku_warn(f"Ignore invalid bar: {bar_datetime}, missing fields!")
+                continue
+
             if last_krecord is not None and bar_datetime == last_datetime:
                 if abs(last_krecord[1] - bar["open"]) / last_krecord[1] > 0.01:
                     hku_error(
@@ -369,7 +373,7 @@ def import_one_stock_data(
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord amount: {last_krecord[5]}, bar: {bar['amount']*0.001}")
                     return 0
-                if ktype == 'DAY' and last_krecord[5] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
+                if ktype == 'DAY' and last_krecord[6] != 0.0 and abs(last_krecord[6] - bar["vol"]) / last_krecord[6] > 0.1:
                     hku_error(
                         f"fetch data from tdx error! {bar_datetime} {ktype} {market}{code} last_krecord count: {last_krecord[6]}, bar: {bar['vol']}")
                     return 0

hikyuu/data/pytdx_weight_to_clickhouse.py

@@ -34,7 +34,7 @@ def pytdx_import_weight_to_clickhouse(pytdx_api, connect, market):
 
     total_count = 0
     update_count = 0
-    stock_list = connect.query(f"select code from `hku_base`.`stock` where market='{market}' and valid=1")
+    stock_list = connect.query(f"select code from `hku_base`.`stock` where market='{market}'")
     stockid_list = stock_list.result_rows
 
     for stockrecord in stockid_list:

hikyuu/data/pytdx_weight_to_mysql.py

@@ -38,7 +38,7 @@ def pytdx_import_weight_to_mysql(pytdx_api, connect, market):
     pytdx_market = to_pytdx_market(market)
 
     total_count = 0
-    cur.execute("select stockid, code from `hku_base`.`stock` where marketid=%s and valid=1" % (marketid))
+    cur.execute("select stockid, code from `hku_base`.`stock` where marketid=%s" % (marketid))
     stockid_list = [x for x in cur.fetchall()]
     cur.close()
 

hikyuu/data/pytdx_weight_to_sqlite.py

@@ -35,7 +35,7 @@ def pytdx_import_weight_to_sqlite(pytdx_api, connect, market):
     pytdx_market = to_pytdx_market(market)
 
     total_count = 0
-    stockid_list = cur.execute("select stockid, code from Stock where marketid=%s and valid=1" % (marketid))
+    stockid_list = cur.execute("select stockid, code from Stock where marketid=%s" % (marketid))
     stockid_list = [x for x in stockid_list]
     cur.close()
 

hikyuu/data/zh_bond10_to_clickhouse.py

@@ -13,7 +13,7 @@ from hikyuu.util import *
 def import_zh_bond10_to_clickhouse(connect):
     sql = "select max(date) from hku_base.zh_bond10"
     start_date = connect.command(sql)
-    if type(start_date) == int:
+    if start_date == 0:
         start_date = "19901219"
     else:
         last_date = start_date