hikyuu 2.6.3__py3-none-win_amd64.whl → 2.6.6__py3-none-win_amd64.whl

hikyuu/gui/HikyuuTDX.py

@@ -22,6 +22,8 @@ import mysql.connector
 from mysql.connector import errorcode
 from mysql.connector.locales.eng import client_error  # 此句仅为pyinstaller打包时能够自动引入
 
+import clickhouse_connect
+
 from hikyuu.gui.data.MainWindow import *
 from hikyuu.gui.data.EscapetimeThread import EscapetimeThread
 from hikyuu.gui.data.UseTdxImportToH5Thread import UseTdxImportToH5Thread
@@ -34,7 +36,7 @@ from hikyuu.gui.data.CollectSpotThread import CollectSpotThread
 from hikyuu.gui.data.SchedImportThread import SchedImportThread
 from hikyuu.gui.spot_server import release_nng_senders
 
-from hikyuu import can_upgrade, get_last_version, fetch_trial_license, view_license
+from hikyuu import can_upgrade, get_last_version, fetch_trial_license, view_license, is_valid_license
 from hikyuu.data import hku_config_template
 from hikyuu.util import *
 
@@ -93,6 +95,7 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
     def getHikyuuConfigFileName(self):
         return self.getUserConfigDir() + '/hikyuu.ini'
 
+    @hku_catch()
     def saveConfig(self):
         if not os.path.lexists(self.getUserConfigDir()):
             os.mkdir(self.getUserConfigDir())
@@ -105,7 +108,10 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
         if current_config.getboolean('hdf5', 'enable', fallback=True):
             data_dir = current_config['hdf5']['dir']
             if not os.path.lexists(data_dir + '/tmp'):
-                os.mkdir(data_dir + '/tmp')
+                try:
+                    os.mkdir(data_dir + '/tmp')
+                except:
+                    pass
             # 此处不能使用 utf-8 参数，否则导致Windows下getBlock无法找到板块分类
             with open(filename, 'w', encoding='utf-8') as f:
                 f.write(
@@ -140,8 +146,13 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
                     )
                 )
 
-        else:
+        elif current_config.getboolean('mysql', 'enable', fallback=True):
             data_dir = current_config['mysql']['tmpdir']
+            if not os.path.lexists(data_dir + '/tmp'):
+                try:
+                    os.mkdir(data_dir + '/tmp')
+                except:
+                    pass
             with open(filename, 'w', encoding="utf-8") as f:
                 f.write(
                     hku_config_template.mysql_template.format(
@@ -178,16 +189,65 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
                         hour2_max=current_config.getint('preload', 'hour2_max', fallback=4096),
                     )
                 )
+        elif current_config.getboolean('clickhouse', 'enable', fallback=True):
+            data_dir = current_config['clickhouse']['tmpdir']
+            if not os.path.lexists(data_dir + '/tmp'):
+                try:
+                    os.mkdir(data_dir + '/tmp')
+                except:
+                    pass
+            with open(filename, 'w', encoding="utf-8") as f:
+                f.write(
+                    hku_config_template.clickhouse_template.format(
+                        dir=data_dir,
+                        quotation_server=current_config.get(
+                            'collect', 'quotation_server', fallback='ipc:///tmp/hikyuu_real.ipc'),
+                        host=current_config['clickhouse']['host'],
+                        port=current_config['clickhouse']['port'],
+                        usr=current_config['clickhouse']['usr'],
+                        pwd=current_config['clickhouse']['pwd'],
+                        day=current_config.getboolean('preload', 'day', fallback=True),
+                        week=current_config.getboolean('preload', 'week', fallback=False),
+                        month=current_config.getboolean('preload', 'month', fallback=False),
+                        quarter=current_config.getboolean('preload', 'quarter', fallback=False),
+                        halfyear=current_config.getboolean('preload', 'halfyear', fallback=False),
+                        year=current_config.getboolean('preload', 'year', fallback=False),
+                        min1=current_config.getboolean('preload', 'min', fallback=False),
+                        min5=current_config.getboolean('preload', 'min5', fallback=False),
+                        min15=current_config.getboolean('preload', 'min15', fallback=False),
+                        min30=current_config.getboolean('preload', 'min30', fallback=False),
+                        min60=current_config.getboolean('preload', 'min60', fallback=False),
+                        hour2=current_config.getboolean('preload', 'hour2', fallback=False),
+                        day_max=current_config.getint('preload', 'day_max', fallback=100000),
+                        week_max=current_config.getint('preload', 'week_max', fallback=100000),
+                        month_max=current_config.getint('preload', 'month_max', fallback=100000),
+                        quarter_max=current_config.getint('preload', 'quarter_max', fallback=100000),
+                        halfyear_max=current_config.getint('preload', 'halfyear_max', fallback=100000),
+                        year_max=current_config.getint('preload', 'year_max', fallback=100000),
+                        min1_max=current_config.getint('preload', 'min_max', fallback=4096),
+                        min5_max=current_config.getint('preload', 'min5_max', fallback=4096),
+                        min15_max=current_config.getint('preload', 'min15_max', fallback=4096),
+                        min30_max=current_config.getint('preload', 'min30_max', fallback=4096),
+                        min60_max=current_config.getint('preload', 'min60_max', fallback=4096),
+                        hour2_max=current_config.getint('preload', 'hour2_max', fallback=4096),
+                    )
+                )
 
-        if not os.path.lexists(data_dir):
-            os.makedirs(data_dir)
+        try:
+            if not data_dir and not os.path.lexists(data_dir):
+                os.makedirs(data_dir)
+        except:
+            pass
 
-        if not os.path.lexists(data_dir + '/block'):
-            current_dir = os.path.dirname(os.path.abspath(__file__))
-            dirname, _ = os.path.split(current_dir)
-            dirname = os.path.join(dirname, 'config/block')
-            shutil.copytree(dirname, data_dir + '/block')
-            os.remove(data_dir + '/block/__init__.py')
+        try:
+            if not data_dir and not os.path.lexists(data_dir + '/block'):
+                current_dir = os.path.dirname(os.path.abspath(__file__))
+                dirname, _ = os.path.split(current_dir)
+                dirname = os.path.join(dirname, 'config/block')
+                shutil.copytree(dirname, data_dir + '/block')
+                os.remove(data_dir + '/block/__init__.py')
+        except:
+            pass
 
     @pyqtSlot()
     def on_save_pushButton_clicked(self):
@@ -267,6 +327,8 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
                     sys.stderr = self._stream
         self.log_textEdit.document().setMaximumBlockCount(1000)
 
+        self.tabWidget.setCurrentIndex(0)
+
         current_dir = os.path.dirname(__file__)
         icon = QIcon(f"{current_dir}/images/hikyuu_small.png")
         star_img = QPixmap(f"{current_dir}/images/star.png")
@@ -309,6 +371,8 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
 
         # 初始化权息与财务数据设置
         self.import_weight_checkBox.setChecked(import_config.getboolean('weight', 'enable', fallback=True))
+        self.import_history_finance_checkBox.setChecked(import_config.getboolean('finance', 'enable', fallback=True))
+        self.import_block_checkBox.setChecked(import_config.getboolean('block', 'enable', fallback=True))
 
         # 初始化通道信目录配置
         tdx_enable = import_config.getboolean('tdx', 'enable', fallback=False)
@@ -350,7 +414,8 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
         if hdf5_enable:
             mysql_enable = False
         self.enable_mysql_radioButton.setChecked(mysql_enable)
-        self.mysql_tmpdir_lineEdit.setText(import_config.get('mysql', 'tmpdir', fallback='c:/stock'))
+        self.mysql_tmpdir_lineEdit.setText(import_config.get('mysql', 'tmpdir', fallback='d:/stock'))
+        self.mysql_tmpdir_pushButton.setEnabled(mysql_enable)
         mysql_ip = import_config.get('mysql', 'host', fallback='127.0.0.1')
         self.mysql_ip_lineEdit.setText(mysql_ip)
         self.mysql_ip_lineEdit.setEnabled(mysql_enable)
@@ -365,6 +430,30 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
         self.mysql_pwd_lineEdit.setEnabled(mysql_enable)
         self.mysql_test_pushButton.setEnabled(mysql_enable)
 
+        # 初始化clickhouse设置
+        clickhouse_enable = import_config.getboolean('clickhouse', 'enable', fallback=False)
+        if (not is_valid_license()) or hdf5_enable or mysql_enable:
+            clickhouse_enable = False
+        self.enable_clickhouse_radioButton.setChecked(clickhouse_enable)
+        self.clickhouse_tmpdir_lineEdit.setText(import_config.get('clickhouse', 'tmpdir', fallback='d:/stock'))
+        self.clickhouse_tmpdir_pushButton.setEnabled(clickhouse_enable)
+        clickhouse_ip = import_config.get('clickhouse', 'host', fallback='127.0.0.1')
+        self.clickhouse_ip_lineEdit.setText(clickhouse_ip)
+        self.clickhouse_ip_lineEdit.setEnabled(clickhouse_enable)
+        clickhouse_port = import_config.get('clickhouse', 'port', fallback='9000')
+        self.clickhouse_port_lineEdit.setText(clickhouse_port)
+        self.clickhouse_port_lineEdit.setEnabled(clickhouse_enable)
+        clickhouse_http_port = import_config.get('clickhouse', 'http_port', fallback='8123')
+        self.clickhouse_http_port_lineEdit.setText(clickhouse_http_port)
+        self.clickhouse_http_port_lineEdit.setEnabled(clickhouse_enable)
+        clickhouse_usr = import_config.get('clickhouse', 'usr', fallback='default')
+        self.clickhouse_usr_lineEdit.setText(clickhouse_usr)
+        self.clickhouse_usr_lineEdit.setEnabled(clickhouse_enable)
+        clickhouse_pwd = import_config.get('clickhouse', 'pwd', fallback='test')
+        self.clickhouse_pwd_lineEdit.setText(clickhouse_pwd)
+        self.clickhouse_pwd_lineEdit.setEnabled(clickhouse_enable)
+        self.clickhouse_test_pushButton.setEnabled(clickhouse_enable)
+
         self.sched_import_timeEdit.setTime(
             datetime.time.fromisoformat(import_config.get('schec', 'time', fallback='18:00'))
         )
@@ -442,6 +531,12 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
         import_config['weight'] = {
             'enable': self.import_weight_checkBox.isChecked(),
         }
+        import_config['finance'] = {
+            'enable': self.import_history_finance_checkBox.isChecked(),
+        }
+        import_config['block'] = {
+            'enable': self.import_block_checkBox.isChecked(),
+        }
         import_config['tdx'] = {'enable': self.tdx_radioButton.isChecked(), 'dir': self.tdx_dir_lineEdit.text()}
         import_config['pytdx'] = {
             'enable': self.pytdx_radioButton.isChecked(),
@@ -460,6 +555,15 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
             'usr': self.mysql_usr_lineEdit.text(),
             'pwd': self.mysql_pwd_lineEdit.text()
         }
+        import_config['clickhouse'] = {
+            'enable': is_valid_license() and self.enable_clickhouse_radioButton.isChecked(),
+            'tmpdir': self.clickhouse_tmpdir_lineEdit.text(),
+            'host': self.clickhouse_ip_lineEdit.text(),
+            'http_port': self.clickhouse_http_port_lineEdit.text(),
+            'port': self.clickhouse_port_lineEdit.text(),
+            'usr': self.clickhouse_usr_lineEdit.text(),
+            'pwd': self.clickhouse_pwd_lineEdit.text()
+        }
         import_config['sched'] = {
             'time': self.sched_import_timeEdit.time().toString(),
         }
@@ -597,23 +701,38 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
     def on_enable_hdf55_radioButton_clicked(self):
         if self.enable_hdf55_radioButton.isChecked():
             self.enable_mysql_radioButton.setChecked(False)
-        self.on_enable_hdf5_or_mysql_toggled()
+            self.enable_clickhouse_radioButton.setChecked(False)
+        self.on_enable_database_toggled(hdf5=True, mysql=False, clickhouse=False)
 
     @pyqtSlot()
     def on_enable_mysql_radioButton_clicked(self):
         if self.enable_mysql_radioButton.isChecked():
             self.enable_hdf55_radioButton.setChecked(False)
-        self.on_enable_hdf5_or_mysql_toggled()
+            self.enable_clickhouse_radioButton.setChecked(False)
+        self.on_enable_database_toggled(hdf5=False, mysql=True, clickhouse=False)
 
-    def on_enable_hdf5_or_mysql_toggled(self):
-        hdf5_enable = self.enable_hdf55_radioButton.isChecked()
-        mysql_enable = not hdf5_enable
-        self.hdf5_dir_lineEdit.setEnabled(hdf5_enable)
-        self.mysql_ip_lineEdit.setEnabled(mysql_enable)
-        self.mysql_port_lineEdit.setEnabled(mysql_enable)
-        self.mysql_usr_lineEdit.setEnabled(mysql_enable)
-        self.mysql_pwd_lineEdit.setEnabled(mysql_enable)
-        self.mysql_test_pushButton.setEnabled(mysql_enable)
+    @pyqtSlot()
+    def on_enable_clickhouse_radioButton_clicked(self):
+        if self.enable_clickhouse_radioButton.isChecked():
+            self.enable_hdf55_radioButton.setChecked(False)
+            self.enable_mysql_radioButton.setChecked(False)
+        self.on_enable_database_toggled(hdf5=False, mysql=False, clickhouse=True)
+
+    def on_enable_database_toggled(self, hdf5, mysql, clickhouse):
+        self.hdf5_dir_lineEdit.setEnabled(hdf5)
+        self.mysql_ip_lineEdit.setEnabled(mysql)
+        self.mysql_port_lineEdit.setEnabled(mysql)
+        self.mysql_usr_lineEdit.setEnabled(mysql)
+        self.mysql_pwd_lineEdit.setEnabled(mysql)
+        self.mysql_test_pushButton.setEnabled(mysql)
+        self.mysql_tmpdir_pushButton.setEnabled(mysql)
+        self.clickhouse_ip_lineEdit.setEnabled(clickhouse)
+        self.clickhouse_http_port_lineEdit.setEnabled(clickhouse)
+        self.clickhouse_port_lineEdit.setEnabled(clickhouse)
+        self.clickhouse_usr_lineEdit.setEnabled(clickhouse)
+        self.clickhouse_pwd_lineEdit.setEnabled(clickhouse)
+        self.clickhouse_test_pushButton.setEnabled(clickhouse)
+        self.clickhouse_tmpdir_pushButton.setEnabled(clickhouse)
 
     @pyqtSlot()
     def on_mysql_tmpdir_pushButton_clicked(self):
@@ -625,6 +744,20 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
             dirname = dlg.selectedFiles()
             self.mysql_tmpdir_lineEdit.setText(dirname[0])
 
+    @pyqtSlot()
+    def on_clickhouse_tmpdir_pushButton_clicked(self):
+        if not is_valid_license():
+            QMessageBox.critical(self, "clickhouse引擎", "需要捐赠授权才能使用clickhouse引擎")
+            return
+
+        dlg = QFileDialog()
+        dlg.setFileMode(QFileDialog.Directory)
+        config = self.getCurrentConfig()
+        dlg.setDirectory(config['clickhouse']['tmpdir'])
+        if dlg.exec_():
+            dirname = dlg.selectedFiles()
+            self.clickhouse_tmpdir_lineEdit.setText(dirname[0])
+
     @pyqtSlot()
     def on_mysql_test_pushButton_clicked(self):
         """测试数据库连接"""
@@ -649,6 +782,25 @@ class MyMainWindow(QMainWindow, Ui_MainWindow):
 
         QMessageBox.about(self, "测试数据库连接", " 连接成功！")
 
+    @pyqtSlot()
+    def on_clickhouse_test_pushButton_clicked(self):
+        """测试数据库连接"""
+        db_config = {
+            'username': self.clickhouse_usr_lineEdit.text(),
+            'password': self.clickhouse_pwd_lineEdit.text(),
+            'host': self.clickhouse_ip_lineEdit.text(),
+            'port': self.clickhouse_http_port_lineEdit.text()
+        }
+
+        try:
+            cnx = clickhouse_connect.get_client(**db_config)
+            cnx.close()
+        except Exception as err:
+            QMessageBox.critical(self, "测试数据库连接失败", str(err))
+            return
+
+        QMessageBox.about(self, "测试数据库连接", " 连接成功！")
+
     def reset_progress_bar(self):
         self.hdf5_weight_label.setText('')
         self.hdf5_day_progressBar.setValue(0)

hikyuu/gui/data/ImportBlockInfoTask.py

@@ -9,6 +9,7 @@ import mysql.connector
 from hikyuu.data.common import MARKET, get_stk_code_name_list
 from hikyuu.data.em_block_to_mysql import em_import_block_to_mysql
 from hikyuu.data.em_block_to_sqlite import em_import_block_to_sqlite
+from hikyuu.data.em_block_to_clickhouse import em_import_block_to_clickhouse
 from hikyuu.util import *
 
 
@@ -29,7 +30,7 @@ class ImportBlockInfoTask:
             sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
             connect = sqlite3.connect(sqlite_file, timeout=1800)
             import_block = em_import_block_to_sqlite
-        else:
+        elif self.config.getboolean('mysql', 'enable', fallback=True):
             db_config = {
                 'user': self.config['mysql']['usr'],
                 'password': self.config['mysql']['pwd'],
@@ -38,6 +39,16 @@ class ImportBlockInfoTask:
             }
             connect = mysql.connector.connect(**db_config)
             import_block = em_import_block_to_mysql
+        elif self.config.getboolean('clickhouse', 'enable', fallback=True):
+            db_config = {
+                'username': self.config['clickhouse']['usr'],
+                'password': self.config['clickhouse']['pwd'],
+                'host': self.config['clickhouse']['host'],
+                'port': self.config['clickhouse']['http_port']
+            }
+            import clickhouse_connect
+            connect = clickhouse_connect.get_client(**db_config)
+            import_block = em_import_block_to_clickhouse
 
         count = 0
         try:

hikyuu/gui/data/ImportHistoryFinanceTask.py

@@ -27,10 +27,11 @@ import shutil
 import hashlib
 import sqlite3
 import mysql.connector
+import clickhouse_connect
 from pytdx.hq import TdxHq_API
 from hikyuu.data.pytdx_finance_to_mysql import history_finance_import_mysql
 from hikyuu.data.pytdx_finance_to_sqlite import history_finance_import_sqlite
-from hikyuu.data.common_pytdx import search_best_tdx
+from hikyuu.data.pytdx_finance_to_clickhouse import history_finance_import_clickhouse
 from hikyuu.util import *
 
 
@@ -46,13 +47,13 @@ class ImportHistoryFinanceTask:
         self.total_count = 0
         self.status = "no run"
 
-    def connect(self):
-        self.api = TdxHq_API()
+    def connect_to_pytdx(self):
+        api = TdxHq_API()
         # 不是所有服务器都提供下载
-        hku_check(self.api.connect('120.76.152.87', 7709), "failed connect pytdx!")
+        hku_check(api.connect('120.76.152.87', 7709), "failed connect pytdx!")
+        return api
 
     def get_list_info(self):
-        result = []
         content = self.api.get_report_file_by_size('tdxfin/gpcw.txt')
         content = content.decode('utf-8')
         content = content.strip().split('\r\n')
@@ -62,72 +63,89 @@ class ImportHistoryFinanceTask:
 
         return [l2d(i.strip().split(',')) for i in content]
 
-    def import_to_db(self, filename):
+    def connect_db(self):
         if self.config.getboolean('hdf5', 'enable', fallback=True):
             sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
-            connect = sqlite3.connect(sqlite_file, timeout=1800)
-            history_finance_import = history_finance_import_sqlite
-        else:
+            self.db_connect = sqlite3.connect(sqlite_file, timeout=1800)
+            self.history_finance_import = history_finance_import_sqlite
+            self.engine = 'hdf5'
+        elif self.config.getboolean('mysql', 'enable', fallback=True):
             db_config = {
                 'user': self.config['mysql']['usr'],
                 'password': self.config['mysql']['pwd'],
                 'host': self.config['mysql']['host'],
                 'port': self.config['mysql']['port']
             }
-            connect = mysql.connector.connect(**db_config)
-            history_finance_import = history_finance_import_mysql
+            self.db_connect = mysql.connector.connect(**db_config)
+            self.history_finance_import = history_finance_import_mysql
+            self.engine = 'mysql'
+        elif self.config.getboolean('clickhouse', 'enable', fallback=True):
+            db_config = {
+                'username': self.config['clickhouse']['usr'],
+                'password': self.config['clickhouse']['pwd'],
+                'host': self.config['clickhouse']['host'],
+                'port': self.config['clickhouse']['http_port']
+            }
+            self.db_connect = clickhouse_connect.get_client(**db_config)
+            self.history_finance_import = history_finance_import_clickhouse
+            self.engine = 'clickhouse'
 
+    def import_to_db(self, filename):
         try:
-            history_finance_import(connect, filename)
+            self.history_finance_import(self.db_connect, filename)
         except Exception as e:
             hku_error(str(e))
-        finally:
-            connect.commit()
-            connect.close()
 
-    def download_file(self, item):
+    def download_file(self, item, dest_dir):
         filename = item['filename']
-        # filesize = item['filesize']
-        # get_report_file_by_size 传入实际的 filesize，实际会出错
-        data = self.api.get_report_file_by_size(f'tdxfin/{filename}', 0)
-        hku_info(f"Download finance file: {filename}")
-        dest_file_name = self.dest_dir + "/" + filename
-        with open(dest_file_name, 'wb') as f:
-            f.write(data)
-        shutil.unpack_archive(dest_file_name, extract_dir=self.dest_dir)
-        self.import_to_db(f'{self.dest_dir}/{filename[0:-4]}.dat')
-        hku_info(f"Import finance file: {self.dest_dir}/{filename[0:-4]}.dat")
+        dest_file_name = dest_dir + "/" + filename
+        need_download = False
+        if not os.path.exists(dest_file_name):
+            need_download = True
+        else:
+            old_md5 = ''
+            with open(dest_file_name, 'rb') as f:
+                old_md5 = hashlib.md5(f.read()).hexdigest()
+            if old_md5 != item['hash']:
+                need_download = True
+
+        if need_download:
+            api = TdxHq_API()
+            # 不是所有服务器都提供下载
+            hku_check(api.connect('120.76.152.87', 7709), "failed connect pytdx!")
+            data = api.get_report_file_by_size(f'tdxfin/{filename}', 0)
+            hku_info(f"Download finance file: {filename}")
+            with open(dest_file_name, 'wb') as f:
+                f.write(data)
+            api.disconnect()
+
+        shutil.unpack_archive(dest_file_name, extract_dir=dest_dir)
+        self.import_to_db(f'{dest_dir}/{filename[0:-4]}.dat')
+        hku_info(f"Import finance file: {dest_dir}/{filename[0:-4]}.dat")
 
     @hku_catch(trace=True)
     def __call__(self):
         self.status = "running"
         capture_multiprocess_all_logger(self.log_queue)
-        self.connect()
+        self.api = self.connect_to_pytdx()
         data_list = self.get_list_info()
         self.total_count = len(data_list)
         count = 0
+        data_list.sort(key=lambda x: x['filename'])
+
+        self.connect_db()
         for item in data_list:
             try:
-                dest_file = '{}/{}'.format(self.dest_dir, item['filename'])
-                if not os.path.exists(dest_file):
-                    self.download_file(item)
-                else:
-                    old_md5 = ''
-                    with open(dest_file, 'rb') as f:
-                        old_md5 = hashlib.md5(f.read()).hexdigest()
-                    if old_md5 != item['hash']:
-                        self.download_file(item)
-                    else:
-                        # 不管是否有变化，都导入一次，以便切换引擎时可以导入
-                        shutil.unpack_archive(dest_file, extract_dir=self.dest_dir)
-                        filename = item['filename']
-                        filename = f'{self.dest_dir}/{filename[0:-4]}.dat'
-                        self.import_to_db(filename)
-                        hku_info(f"Import finance file: {filename}")
+                self.download_file(item, self.dest_dir)
                 count += 1
                 self.queue.put([self.task_name, None, None, int(100 * count / self.total_count), self.total_count])
             except Exception as e:
                 hku_error(str(e))
+        if self.engine == 'clickhouse':
+            hku_run_ignore_exception(self.db_connect.command, "OPTIMIZE TABLE hku_base.historyfinance FINAL")
+        self.db_connect.close()
+        self.api.disconnect()
+
         self.queue.put([self.task_name, None, None, None, self.total_count])
         self.status = "finished"
 
@@ -138,6 +156,6 @@ if __name__ == "__main__":
     this_dir = os.path.expanduser('~') + '/.hikyuu'
     import_config = ConfigParser()
     import_config.read(this_dir + '/importdata-gui.ini', encoding='utf-8')
-    task = ImportHistoryFinanceTask(None, None, None, "c:\\stock")
+    task = ImportHistoryFinanceTask(None, None, import_config, "/Users/fasiondog/stock")
     task()
     print("over!")

hikyuu/gui/data/ImportPytdxTimeToH5Task.py

@@ -25,9 +25,11 @@
 import logging
 import sqlite3
 import mysql.connector
+import clickhouse_connect
 from pytdx.hq import TdxHq_API
 from hikyuu.data.pytdx_to_h5 import import_time as h5_import_time
 from hikyuu.data.pytdx_to_mysql import import_time as mysql_import_time
+from hikyuu.data.pytdx_to_clickhouse import import_time as clickhouse_import_time
 from hikyuu.util import *
 
 
@@ -64,7 +66,7 @@ class ImportPytdxTimeToH5:
             sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
             connect = sqlite3.connect(sqlite_file, timeout=1800)
             import_time = h5_import_time
-        else:
+        elif self.config.getboolean('mysql', 'enable', fallback=True):
             db_config = {
                 'user': self.config['mysql']['usr'],
                 'password': self.config['mysql']['pwd'],
@@ -73,6 +75,15 @@ class ImportPytdxTimeToH5:
             }
             connect = mysql.connector.connect(**db_config)
             import_time = mysql_import_time
+        elif self.config.getboolean('clickhouse', 'enable', fallback=True):
+            db_config = {
+                'username': self.config['clickhouse']['usr'],
+                'password': self.config['clickhouse']['pwd'],
+                'host': self.config['clickhouse']['host'],
+                'port': self.config['clickhouse']['http_port']
+            }
+            connect = clickhouse_connect.get_client(**db_config)
+            import_time = clickhouse_import_time
 
         count = 0
         try:
@@ -84,10 +95,11 @@ class ImportPytdxTimeToH5:
                 connect, self.market, self.quotations, api, self.dest_dir, max_days=self.max_days, progress=progress
             )
             self.logger.info("导入 {} 分时记录数: {}".format(self.market, count))
+            api.disconnect()
         except Exception as e:
             self.logger.error(e)
         finally:
-            connect.commit()
+            api.close()
             connect.close()
 
         self.queue.put([self.task_name, self.market, 'TIME', None, count])

hikyuu/gui/data/ImportPytdxToH5Task.py

@@ -28,6 +28,7 @@ import mysql.connector
 from pytdx.hq import TdxHq_API
 from hikyuu.data.pytdx_to_h5 import import_data as h5_import_data
 from hikyuu.data.pytdx_to_mysql import import_data as mysql_import_data
+from hikyuu.data.pytdx_to_clickhouse import import_data as clickhouse_import_data
 from hikyuu.util import *
 
 
@@ -66,16 +67,28 @@ class ImportPytdxToH5:
             connect = sqlite3.connect(sqlite_file, timeout=1800)
             import_data = h5_import_data
             self.logger.debug('use hdf5 import kdata')
-        else:
+        elif self.config.getboolean('mysql', 'enable', fallback=True):
             db_config = {
                 'user': self.config['mysql']['usr'],
                 'password': self.config['mysql']['pwd'],
                 'host': self.config['mysql']['host'],
-                'port': self.config['mysql']['port']
+                'port': int(self.config['mysql']['port'])
             }
             connect = mysql.connector.connect(**db_config)
             import_data = mysql_import_data
             self.logger.debug('use mysql import kdata')
+        elif self.config.getboolean('clickhouse', 'enable', fallback=True):
+            db_config = {
+                'username': self.config['clickhouse']['usr'],
+                'password': self.config['clickhouse']['pwd'],
+                'host': self.config['clickhouse']['host'],
+                'port': int(self.config['clickhouse']['http_port']),
+                'send_receive_timeout': 60,
+            }
+            import clickhouse_connect
+            connect = clickhouse_connect.get_client(**db_config)
+            import_data = clickhouse_import_data
+            self.logger.debug('use clickhouse import kdata')
 
         count = 0
         try:
@@ -87,11 +100,12 @@ class ImportPytdxToH5:
                 connect, self.market, self.ktype, self.quotations, api, self.dest_dir, self.startDatetime, progress
             )
             self.logger.info("导入 {} {} 记录数: {}".format(self.market, self.ktype, count))
+            api.disconnect()
         except Exception as e:
             self.logger.error("ImportPytdxToH5Task failed! {}".format(e))
             # self.queue.put([self.task_name, self.market, self.ktype, str(e), count])
         finally:
-            connect.commit()
+            api.close()
             connect.close()
 
         self.queue.put([self.task_name, self.market, self.ktype, None, count])

hikyuu/gui/data/ImportPytdxTransToH5Task.py

@@ -28,6 +28,7 @@ import mysql.connector
 from pytdx.hq import TdxHq_API
 from hikyuu.data.pytdx_to_h5 import import_trans as h5_import_trans
 from hikyuu.data.pytdx_to_mysql import import_trans as mysql_import_trans
+from hikyuu.data.pytdx_to_clickhouse import import_trans as clickhouse_import_trans
 from hikyuu.util import *
 
 
@@ -64,7 +65,7 @@ class ImportPytdxTransToH5:
             sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
             connect = sqlite3.connect(sqlite_file, timeout=1800)
             import_trans = h5_import_trans
-        else:
+        elif self.config.getboolean('mysql', 'enable', fallback=True):
             db_config = {
                 'user': self.config['mysql']['usr'],
                 'password': self.config['mysql']['pwd'],
@@ -73,6 +74,16 @@ class ImportPytdxTransToH5:
             }
             connect = mysql.connector.connect(**db_config)
             import_trans = mysql_import_trans
+        elif self.config.getboolean('clickhouse', 'enable', fallback=True):
+            db_config = {
+                'username': self.config['clickhouse']['usr'],
+                'password': self.config['clickhouse']['pwd'],
+                'host': self.config['clickhouse']['host'],
+                'port': self.config['clickhouse']['http_port']
+            }
+            import clickhouse_connect
+            connect = clickhouse_connect.get_client(**db_config)
+            import_trans = clickhouse_import_trans
 
         count = 0
         try:
@@ -84,10 +95,11 @@ class ImportPytdxTransToH5:
                 connect, self.market, self.quotations, api, self.dest_dir, max_days=self.max_days, progress=progress
             )
             self.logger.info("导入 {} 分笔记录数: {}".format(self.market, count))
+            api.disconnect()
         except Exception as e:
             self.logger.error(e)
         finally:
-            connect.commit()
+            api.close()
             connect.close()
 
         self.queue.put([self.task_name, self.market, 'TRANS', None, count])