hikyuu 2.6.3__py3-none-win_amd64.whl → 2.6.6__py3-none-win_amd64.whl

hikyuu/data/pytdx_weight_to_clickhouse.py

@@ -0,0 +1,191 @@
+# coding:utf-8
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2010-2019 fasiondog/hikyuu
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from pytdx.hq import TDXParams
+from hikyuu.util import *
+from hikyuu.data.common_pytdx import to_pytdx_market
+
+
+@hku_catch(trace=True)
+def pytdx_import_weight_to_clickhouse(pytdx_api, connect, market):
+    """从 pytdx 导入权息数据"""
+    pytdx_market = to_pytdx_market(market)
+
+    total_count = 0
+    update_count = 0
+    stock_list = connect.query(f"select code from `hku_base`.`stock` where market='{market}' and valid=1")
+    stockid_list = stock_list.result_rows
+
+    for stockrecord in stockid_list:
+        code = stockrecord[0]
+
+        # 获取当前数据库中最后的一条权息记录的总股本和流通股本
+        sql = f"select date, countAsGift, countForSell, priceForSell, bonus, countOfIncreasement, totalCount, freeCount, suogu from `hku_base`.`stkweight` where market='{market}' and code='{code}' order by date desc limit 1"
+        tmp = connect.query(sql)
+        a = tmp.result_rows
+        last_db_weight = None
+        if a:
+            a = list(a[0])
+            last_db_weight = a
+            db_last_date = a[0]
+            last_total_count, last_free_count = a[6:8]
+        else:
+            last_db_weight = None
+            db_last_date, last_total_count, last_free_count = (0, 0, 0)
+
+        xdxr_list = pytdx_api.get_xdxr_info(pytdx_market, code)
+        update_last_db_weight = False
+        new_last_db_weight = last_db_weight
+        records = {}
+        for xdxr in xdxr_list:
+            try:
+                date = xdxr['year'] * 10000 + xdxr['month'] * 100 + xdxr['day']
+                if date < db_last_date:
+                    continue
+                if date == db_last_date and new_last_db_weight is not None:
+                    if xdxr['songzhuangu'] is not None:
+                        new_last_db_weight[1] = 10000 * xdxr['songzhuangu']
+                        update_last_db_weight = True
+                    if xdxr['peigu'] is not None:
+                        new_last_db_weight[2] = 10000 * xdxr['peigu']
+                        update_last_db_weight = True
+                    if xdxr['peigujia'] is not None:
+                        new_last_db_weight[3] = 1000 * xdxr['peigujia']
+                        update_last_db_weight = True
+                    if xdxr['fenhong'] is not None:
+                        new_last_db_weight[4] = 1000 * xdxr['fenhong']
+                        update_last_db_weight = True
+                    if xdxr['houzongguben'] is not None:
+                        new_last_db_weight[6] = xdxr['houzongguben']
+                        update_last_db_weight = True
+                        last_total_count = new_last_db_weight[6]
+                    if xdxr['panhouliutong'] is not None:
+                        new_last_db_weight[7] = xdxr['panhouliutong']
+                        update_last_db_weight = True
+                        last_free_count = new_last_db_weight[7]
+                    if xdxr['suogu'] is not None:
+                        # etf 扩缩股
+                        new_last_db_weight[8] = xdxr['suogu']
+                        update_last_db_weight = True
+                    continue
+                if date not in records:
+                    records[date] = [
+                        date,
+                        10000 * xdxr['songzhuangu'] if xdxr['songzhuangu'] is not None else 0,  # countAsGift
+                        10000 * xdxr['peigu'] if xdxr['peigu'] is not None else 0,  # countForSell
+                        1000 * xdxr['peigujia'] if xdxr['peigujia'] is not None else 0,  # priceForSell
+                        1000 * xdxr['fenhong'] if xdxr['fenhong'] is not None else 0,  # bonus
+                        0,  # countOfIncreasement, pytdx 不区分送股和转增股，统一记在送股
+                        xdxr['houzongguben'] if xdxr['houzongguben'] is not None else last_total_count,  # totalCount
+                        xdxr['panhouliutong'] if xdxr['panhouliutong'] is not None else last_free_count,  # freeCount
+                        xdxr["suogu"] if xdxr["suogu"] is not None else 0
+                    ]
+                else:
+                    if xdxr['songzhuangu'] is not None:
+                        records[date][1] = 10000 * xdxr['songzhuangu']
+                    if xdxr['peigu'] is not None:
+                        records[date][2] = 10000 * xdxr['peigu']
+                    if xdxr['peigujia'] is not None:
+                        records[date][3] = 1000 * xdxr['peigujia']
+                    if xdxr['fenhong'] is not None:
+                        records[date][4] = 1000 * xdxr['fenhong']
+                    if xdxr['houzongguben'] is not None:
+                        records[date][6] = xdxr['houzongguben']
+                    if xdxr['panhouliutong'] is not None:
+                        records[date][7] = xdxr['panhouliutong']
+                    if xdxr['suogu'] is not None:
+                        records[date][8] = xdxr['suogu']
+                if xdxr['houzongguben'] is not None:
+                    last_total_count = xdxr['houzongguben']
+                if xdxr['panhouliutong'] is not None:
+                    last_free_count = xdxr['panhouliutong']
+            except Exception as e:
+                print(e)
+                print("{}{} xdxr: {} last_db_weigth:{}".format(market, code, xdxr, new_last_db_weight))
+                raise e
+
+        if update_last_db_weight and new_last_db_weight != last_db_weight:
+            v = new_last_db_weight
+            sql = f"delete from hku_base.stkweight where market='{market}' and code='{code}' and date={v[0]}"
+            connect.command(sql)
+            records[v[0]] = v
+            update_count += 1
+
+        if records:
+            insert_records = []
+            for v in records.values():
+                tmp = [market, code]
+                tmp.extend(v)
+                insert_records.append(tmp)
+            ic = connect.create_insert_context(table='stkweight', database='hku_base',
+                                               column_names=['market', 'code', 'date', 'countAsGift', 'countForSell',
+                                                             'priceForSell', 'bonus', 'countOfIncreasement', 'totalCount', 'freeCount', 'suogu'],
+                                               data=insert_records)
+            connect.insert(context=ic)
+            total_count += len(records)
+
+    total_count = total_count - update_count
+    return total_count if total_count > 0 else 0
+
+
+if __name__ == '__main__':
+    import os
+    from configparser import ConfigParser
+    dev_config = ConfigParser()
+    dev_config.read(os.path.expanduser("~") + '/workspace/dev.ini')
+    db = 'clickhouse54-http'
+    user = dev_config.get(db, 'user')
+    password = dev_config.get(db, 'pwd')
+    host = dev_config.get(db, 'host')
+    port = dev_config.getint(db, 'port')
+
+    import clickhouse_connect
+    client = clickhouse_connect.get_client(
+        host=host, username=user, password=password)
+
+    # sql = "select date, countAsGift, countForSell, priceForSell, bonus, countOfIncreasement, totalCount, freeCount, suogu from `hku_base`.`stkweight` where market='SH' and code='000001' order by date desc limit 1"
+    # result = client.query(sql)
+    # print(result.result_rows)
+    # client.close()
+
+    tdx_server = '180.101.48.170'
+    tdx_port = 7709
+    quotations = ['stock', 'fund']
+
+    from pytdx.hq import TdxHq_API, TDXParams
+    api = TdxHq_API()
+    api.connect(tdx_server, tdx_port)
+
+    import time
+    starttime = time.time()
+    print("开始导入")
+    print("导入数量: {}".format(pytdx_import_weight_to_clickhouse(api, client, 'SH')))
+
+    api.disconnect()
+    client.close()
+
+    endtime = time.time()
+    print("\nTotal time:")
+    print("%.2fs" % (endtime - starttime))
+    print("%.2fm" % ((endtime - starttime) / 60))

hikyuu/data/sqlite_upgrade/0028.sql

@@ -0,0 +1,97 @@
+BEGIN TRANSACTION;
+DELETE FROM holiday;
+INSERT INTO holiday (`id`, `date`) VALUES (1, 20210101);
+INSERT INTO holiday (`id`, `date`) VALUES (2, 20210211);
+INSERT INTO holiday (`id`, `date`) VALUES (3, 20210212);
+INSERT INTO holiday (`id`, `date`) VALUES (4, 20210215);
+INSERT INTO holiday (`id`, `date`) VALUES (5, 20210216);
+INSERT INTO holiday (`id`, `date`) VALUES (6, 20210217);
+INSERT INTO holiday (`id`, `date`) VALUES (7, 20210405);
+INSERT INTO holiday (`id`, `date`) VALUES (8, 20210501);
+INSERT INTO holiday (`id`, `date`) VALUES (9, 20210503);
+INSERT INTO holiday (`id`, `date`) VALUES (10, 20210504);
+INSERT INTO holiday (`id`, `date`) VALUES (11, 20210505);
+INSERT INTO holiday (`id`, `date`) VALUES (12, 20210614);
+INSERT INTO holiday (`id`, `date`) VALUES (13, 20210920);
+INSERT INTO holiday (`id`, `date`) VALUES (14, 20210921);
+INSERT INTO holiday (`id`, `date`) VALUES (15, 20211001);
+INSERT INTO holiday (`id`, `date`) VALUES (16, 20211004);
+INSERT INTO holiday (`id`, `date`) VALUES (17, 20211005);
+INSERT INTO holiday (`id`, `date`) VALUES (18, 20211006);
+INSERT INTO holiday (`id`, `date`) VALUES (19, 20211007);
+INSERT INTO holiday (`id`, `date`) VALUES (20, 20220103);
+INSERT INTO holiday (`id`, `date`) VALUES (21, 20220131);
+INSERT INTO holiday (`id`, `date`) VALUES (22, 20220201);
+INSERT INTO holiday (`id`, `date`) VALUES (23, 20220202);
+INSERT INTO holiday (`id`, `date`) VALUES (24, 20220203);
+INSERT INTO holiday (`id`, `date`) VALUES (25, 20220204);
+INSERT INTO holiday (`id`, `date`) VALUES (26, 20220205);
+INSERT INTO holiday (`id`, `date`) VALUES (27, 20220404);
+INSERT INTO holiday (`id`, `date`) VALUES (28, 20220405);
+INSERT INTO holiday (`id`, `date`) VALUES (29, 20220502);
+INSERT INTO holiday (`id`, `date`) VALUES (30, 20220503);
+INSERT INTO holiday (`id`, `date`) VALUES (31, 20220504);
+INSERT INTO holiday (`id`, `date`) VALUES (32, 20220603);
+INSERT INTO holiday (`id`, `date`) VALUES (33, 20220912);
+INSERT INTO holiday (`id`, `date`) VALUES (34, 20221003);
+INSERT INTO holiday (`id`, `date`) VALUES (35, 20221004);
+INSERT INTO holiday (`id`, `date`) VALUES (36, 20221005);
+INSERT INTO holiday (`id`, `date`) VALUES (37, 20221006);
+INSERT INTO holiday (`id`, `date`) VALUES (38, 20221007);
+INSERT INTO holiday (`id`, `date`) VALUES (39, 20230102);
+INSERT INTO holiday (`id`, `date`) VALUES (40, 20230123);
+INSERT INTO holiday (`id`, `date`) VALUES (41, 20230124);
+INSERT INTO holiday (`id`, `date`) VALUES (42, 20230125);
+INSERT INTO holiday (`id`, `date`) VALUES (43, 20230126);
+INSERT INTO holiday (`id`, `date`) VALUES (44, 20230127);
+INSERT INTO holiday (`id`, `date`) VALUES (45, 20230405);
+INSERT INTO holiday (`id`, `date`) VALUES (46, 20230501);
+INSERT INTO holiday (`id`, `date`) VALUES (47, 20230502);
+INSERT INTO holiday (`id`, `date`) VALUES (48, 20230503);
+INSERT INTO holiday (`id`, `date`) VALUES (49, 20230622);
+INSERT INTO holiday (`id`, `date`) VALUES (50, 20230623);
+INSERT INTO holiday (`id`, `date`) VALUES (51, 20230929);
+INSERT INTO holiday (`id`, `date`) VALUES (52, 20231002);
+INSERT INTO holiday (`id`, `date`) VALUES (53, 20231003);
+INSERT INTO holiday (`id`, `date`) VALUES (54, 20231004);
+INSERT INTO holiday (`id`, `date`) VALUES (55, 20231005);
+INSERT INTO holiday (`id`, `date`) VALUES (56, 20231006);
+INSERT INTO holiday (`id`, `date`) VALUES (57, 20240101);
+INSERT INTO holiday (`id`, `date`) VALUES (58, 20240212);
+INSERT INTO holiday (`id`, `date`) VALUES (59, 20240213);
+INSERT INTO holiday (`id`, `date`) VALUES (60, 20240214);
+INSERT INTO holiday (`id`, `date`) VALUES (61, 20240215);
+INSERT INTO holiday (`id`, `date`) VALUES (62, 20240216);
+INSERT INTO holiday (`id`, `date`) VALUES (63, 20240404);
+INSERT INTO holiday (`id`, `date`) VALUES (64, 20240405);
+INSERT INTO holiday (`id`, `date`) VALUES (65, 20240501);
+INSERT INTO holiday (`id`, `date`) VALUES (66, 20240502);
+INSERT INTO holiday (`id`, `date`) VALUES (67, 20240503);
+INSERT INTO holiday (`id`, `date`) VALUES (68, 20240610);
+INSERT INTO holiday (`id`, `date`) VALUES (69, 20240916);
+INSERT INTO holiday (`id`, `date`) VALUES (70, 20240917);
+INSERT INTO holiday (`id`, `date`) VALUES (71, 20241001);
+INSERT INTO holiday (`id`, `date`) VALUES (72, 20241002);
+INSERT INTO holiday (`id`, `date`) VALUES (73, 20241003);
+INSERT INTO holiday (`id`, `date`) VALUES (74, 20241004);
+INSERT INTO holiday (`id`, `date`) VALUES (75, 20241007);
+INSERT INTO holiday (`id`, `date`) VALUES (76, 20250101);
+INSERT INTO holiday (`id`, `date`) VALUES (77, 20250128);
+INSERT INTO holiday (`id`, `date`) VALUES (78, 20250129);
+INSERT INTO holiday (`id`, `date`) VALUES (79, 20250130);
+INSERT INTO holiday (`id`, `date`) VALUES (80, 20250131);
+INSERT INTO holiday (`id`, `date`) VALUES (81, 20250203);
+INSERT INTO holiday (`id`, `date`) VALUES (82, 20250204);
+INSERT INTO holiday (`id`, `date`) VALUES (83, 20250404);
+INSERT INTO holiday (`id`, `date`) VALUES (84, 20250501);
+INSERT INTO holiday (`id`, `date`) VALUES (85, 20250502);
+INSERT INTO holiday (`id`, `date`) VALUES (86, 20250505);
+INSERT INTO holiday (`id`, `date`) VALUES (87, 20250602);
+INSERT INTO holiday (`id`, `date`) VALUES (88, 20251001);
+INSERT INTO holiday (`id`, `date`) VALUES (89, 20251002);
+INSERT INTO holiday (`id`, `date`) VALUES (90, 20251003);
+INSERT INTO holiday (`id`, `date`) VALUES (91, 20251006);
+INSERT INTO holiday (`id`, `date`) VALUES (92, 20251007);
+INSERT INTO holiday (`id`, `date`) VALUES (93, 20251008);
+UPDATE `version` set `version` = 28;
+COMMIT;