database-smartools 0.0.1__py3-none-any.whl

database_smartools/database/postgre_extend.py

@@ -0,0 +1,396 @@
+# -*- coding: utf-8 -*-
+"""
+@项目名称 : python-main
+@文件名称 : postgre_extend.py
+@创建人   : zhongbinjie
+@创建时间 : 2025/6/7 19:06
+@文件说明 : 
+@企业名称 : 深圳市赢和信息技术有限公司
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+
+from sqlalchemy import create_engine
+import cx_Oracle
+from utils import config, logger, texter
+import numpy as np
+
+# 获取数据池
+cx_Oracle.init_oracle_client(lib_dir=config.MAP['db_lib'])  # 替换为你的Instant Client路径
+POOL_MAP = {}
+
+
+def get_pool():
+    pool = cx_Oracle.SessionPool(user=config.MAP['db_user'],
+                                 password=config.MAP['db_password'],
+                                 dsn=cx_Oracle.makedsn(config.MAP['db_host'], config.MAP['db_port'],
+                                                       config.MAP['db_name']),
+                                 min=2, max=15, increment=1, threaded=False)
+    return pool
+
+
+def get_pool_by_key(db_key, refresh=False):
+    global POOL_MAP
+    scan_db_conf = f"""
+	SELECT user_name,pass_word,url FROM task_template_data_source WHERE db_key = :db_key
+	"""
+    params = {"db_key": db_key}
+    pool = get_pool()
+    conn, message = get_conn(pool)
+    select_result, select_message = select(conn, scan_db_conf, params)
+    if not select_result:
+        return None, select_message
+    db_conf = select_result['data'][0]
+    user_name = db_conf['user_name']
+    pass_word = db_conf['pass_word']
+    url = db_conf['url']
+
+    url_map = texter.parse_jdbc_url(url)
+    pool = None
+    if db_key not in POOL_MAP.keys() or refresh:
+        if url_map['type'] == 'oracle':
+            try:
+                pool = cx_Oracle.SessionPool(user=user_name,
+                                             password=pass_word,
+                                             dsn=cx_Oracle.makedsn(url_map['host'], url_map['port'],
+                                                                   url_map['database']),
+                                             min=2, max=15, increment=1, threaded=False)
+            except Exception as e:
+                message = f"实例化数据库连接池失败，错误信息：{e}"
+                logger.error(message)
+
+                return None, message
+
+        POOL_MAP[db_key] = {
+            "pool": pool,
+            "db_type": url_map['type']
+        }
+        logger.debug(f"创建新的连接池，db_key: {db_key}")
+    else:
+        pool = POOL_MAP[db_key][0]
+        logger.debug(f"使用已有连接池，db_key: {db_key}")
+    return pool, ""
+
+
+def close_pool(pool):
+    pool.close()
+    logger.debug(f"关闭连接池")
+
+
+def get_conn(pool):
+    message = ""
+    try:
+        conn = pool.acquire()
+        logger.debug(f"从连接池中获取数据库连接")
+        return conn, message
+    except Exception as e:
+        message = f'获取数据连接异常，错误信息：{e}'
+        logger.error(f'获取数据连接异常，错误信息：{e}')
+        return None, message
+
+
+def get_conn_map(pool_map):
+    message = ""
+    try:
+        conn = pool_map['pool'].acquire()
+        db_type = pool_map['db_type']
+        conn_map = {
+            "conn": conn,
+            "db_type": db_type
+        }
+        logger.debug(f"从连接池中获取数据库连接和数据库类型")
+        return conn_map, message
+    except Exception as e:
+        message = f'获取数据连接异常，错误信息：{e}'
+        logger.error(f'获取数据连接异常，错误信息：{e}')
+        return None, message
+
+
+def get_engine():
+    url = f'oracle+cx_oracle://{config.DB_USER}:{config.DB_PASSWORD}@{config.DB_HOST}:{config.DB_PORT}/?service_name={config.DB_NAME}'
+    engine = create_engine(url,
+                           pool_size=30,
+                           max_overflow=20,
+                           pool_timeout=30,
+                           pool_recycle=1800,
+                           pool_pre_ping=True)
+
+    return engine
+
+
+def close_engine(engine):
+    engine.dispose()
+
+
+def close_conn(pool, conn):
+    if isinstance(pool, dict):
+        pool = pool['pool']
+
+    if isinstance(conn, dict):
+        conn = conn['conn']
+    pool.release(conn)
+    logger.debug(f"从连接池中释放数据库连接")
+    return conn
+
+
+def commit(conn):
+    if isinstance(conn, dict):
+        conn = conn['conn']
+    conn.commit()
+    logger.debug(f"数据库连接-提交事务")
+    return
+
+
+def rollback(conn):
+    if isinstance(conn, dict):
+        conn = conn['conn']
+    conn.rollback()
+    logger.debug(f"数据库连接-回滚事务")
+    return
+
+
+def select(conn, sql, params=None):
+    result = None
+    message = ""
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                rs = cursor.execute(sql, params).fetchall()
+                result = {
+                    'data': rs,
+                    'desc': cursor.description
+                }
+            else:
+                rs = cursor.execute(sql).fetchall()
+                result = {
+                    'data': rs,
+                    'desc': cursor.description
+                }
+
+            logger.debug(f"[数据查询]sql: {sql}, 参数列表: {params}")
+
+        except Exception as e:
+            message = f"[数据查询]异常：{e}"
+            logger.error(message)
+            return None, message
+
+    # print(result)
+    data = []
+    for row in result['data']:
+        item_map = {}
+        for index, item in enumerate(row):
+            if result['desc'][index][1] == cx_Oracle.DATETIME and item is not None:
+                item = item.strftime("%Y-%m-%d %H:%M:%S")
+
+            item_map[result['desc'][index][0].lower()] = item
+        data.append(item_map)
+    result['data'] = data
+    return result, message
+
+
+def select_by_map(conn_map, sql, params=None):
+    result = None
+    message = ""
+    conn = conn_map['conn']
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                rs = cursor.execute(sql, params).fetchall()
+                result = {
+                    'data': rs,
+                    'desc': cursor.description
+                }
+            else:
+                rs = cursor.execute(sql).fetchall()
+                result = {
+                    'data': rs,
+                    'desc': cursor.description
+                }
+
+            logger.debug(f"[数据查询]sql: {sql}, 参数列表: {params}")
+
+            column_types = []
+            for desc in result['desc']:
+                column_types.append(desc[1])
+            result['data_types'] = column_types
+
+        except Exception as e:
+            message = f"[数据查询]异常：{e}"
+            logger.error(message)
+            return None, message
+
+    # print(result)
+    data = []
+    for row in result['data']:
+        item_map = {}
+        for index, item in enumerate(row):
+            # if result['desc'][index][1] == cx_Oracle.DATETIME and item is not None:
+            # 	item = item.strftime("%Y-%m-%d")
+
+            if result['desc'][index][1] == cx_Oracle.TIMESTAMP and item is not None:
+                item = item.strftime("%Y-%m-%d %H:%M:%S")
+
+            item_map[result['desc'][index][0].lower()] = item
+
+        data.append(item_map)
+    result['data'] = data
+    return result, message
+
+
+def delete(conn, sql, params=None):
+    message = ""
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            logger.debug(f"[数据删除]sql: {sql}, 参数列表: {params}")
+        except Exception as e:
+            message = f"[数据删除]异常：{e}"
+            logger.error(message)
+            return False, message
+
+    return True, message
+
+
+def delete_by_map(conn_map, sql, params=None):
+    message = ""
+    conn = conn_map['conn']
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            logger.debug(f"[数据删除]sql: {sql}, 参数列表: {params}")
+        except Exception as e:
+            message = f"[数据删除]异常：{e}"
+            logger.error(message)
+            return False, message
+
+    return True, message
+
+
+def update(conn, sql, params=None):
+    message = ""
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            logger.debug(f"[数据更新]sql: {sql}, 参数列表: {params}")
+        except Exception as e:
+            message = f"[数据更新]异常：{e}"
+            logger.error(message)
+            return False, message
+    return True, message
+
+
+def update_by_map(conn_map, sql, params=None):
+    message = ""
+    conn = conn_map['conn']
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            logger.debug(f"[数据更新]sql: {sql}, 参数列表: {params}")
+        except Exception as e:
+            message = f"[数据更新]异常：{e}"
+            logger.info(message)
+            return False, message
+    return True, message
+
+
+def insert(conn, sql, params=None):
+    message = ""
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            logger.debug(f"[数据库新增]sql: {sql}, 参数列表: {params}")
+
+        except Exception as e:
+            message = f"[数据库新增]异常：{e}"
+            logger.error(message)
+            return False, message
+
+    return True, message
+
+
+def insert_by_map(conn_map, sql, params=None):
+    message = ""
+    conn = conn_map['conn']
+    with conn.cursor() as cursor:
+        try:
+            if params:
+                cursor.execute(sql, params)
+            else:
+                cursor.execute(sql)
+
+            logger.debug(f"[数据库新增]sql: {sql}, 参数列表: {params}")
+
+        except Exception as e:
+            message = f"[数据库新增]异常：{e}"
+            logger.error(message)
+            return False, message
+
+    return True, message
+
+
+def insert_by_dataframe(conn_map, df, table_name):
+    message = ""
+    conn = conn_map
+    if isinstance(conn_map, dict):
+        conn = conn_map['conn']
+
+    columns = df.columns.tolist()  # 获得dataframe的列名
+    data_to_insert = [tuple(row) for row in df.to_numpy()]  # dataframe转换numpy
+    print(data_to_insert)
+    with conn.cursor() as cursor:
+        try:
+            sql = df_insert2db(columns, save_table_name=table_name)
+            logger.debug(f"[数据库新增][dataframe]sql: {sql}")
+            batch_size = int(config.MAP['batch_size'])
+            for i in range(0, len(data_to_insert), batch_size):
+                batch_data = data_to_insert[i: i + batch_size]
+                print(batch_data)
+                logger.debug(f"[分批写入][dataframe]当前载入至第{min(i + batch_size, len(data_to_insert))}行。")
+                cursor.executemany(sql, batch_data)
+                conn.commit()  # 批量执行query_sql, 每批次提交一次
+
+        except Exception as e:
+            message = f"[数据库新增][dataframe]异常：{e}"
+            logger.error(message)
+            conn.rollback()
+            return False, message
+
+    return True, message
+
+
+def df_insert2db(columns, save_table_name):
+    """
+    Args:
+        columns: 以列表的形式
+        save_table_name: 保存table的名字
+    Returns:直接的insert到Oracle的sql语句，字符串格式
+    """
+    s = ''
+    for i in range(len(columns)):
+        s = s + columns[i] + '-'
+    sql_columns = s.replace('-', ',')[:-1]  # 不要最后输出的逗号
+    sql_number_str = ''
+    for i in range(1, len(columns) + 1, 1):
+        sql_number_str = sql_number_str + ':' + str(i) + ','  # ':1,:2,:3,:4,:5,:6'
+    sql_number_str = sql_number_str[:-1]
+    query_sql = 'insert into ' + save_table_name + '(' + sql_columns + ') values(' + sql_number_str + ') '
+    return query_sql

database_smartools/main.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+"""
+@项目名称 : python-main
+@文件名称 : main.py
+@创建人   : zhongbinjie
+@创建时间 : 2025/6/7 19:06
+@文件说明 : 
+@企业名称 : 深圳市赢和信息技术有限公司
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+import multiprocessing
+
+import uvicorn
+import sys
+from fastapi import FastAPI, Request
+from utils import config, file, logger, http, texter, output, timer
+from api import etl
+import os
+
+app = FastAPI()
+env = "dev"
+res_dir = os.path.dirname(__file__)
+
+if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
+    BASE_DIR = os.path.dirname(res_dir)
+    print("当前工作路径：", BASE_DIR)
+else:
+    BASE_DIR = os.path.dirname(__file__)
+    print("当前工作路径：", BASE_DIR)
+
+workspace = BASE_DIR
+@app.get("/")
+def read_root(request: Request):
+    return
+
+@app.post("/logRefresh")
+def refresh_log():
+    logger.Logger()._check_and_rotate_log(refresh=True)
+    return http.ResponseUtil.success()
+
+@app.post("/configRefresh")
+def refresh_config():
+    config.Config(res_dir + '/conf.ini', 'UTF-8', env)
+    config.add_conf('env', env)
+    config.add_conf('root', workspace)
+    logger.Logger()._check_and_rotate_log(refresh=True)
+    return http.ResponseUtil.success()
+
+# uvicorn提供服务化
+if __name__ == '__main__':
+    print(
+    rf"""
+__   __ _   _  _____  ___  _   _ 
+\ \ / /| | | ||  ___||_ _|| \ | |
+ \ V / | |_| || |_    | | |  \| |
+  | |  |  _  ||  _|   | | | |\  |
+  |_|  |_| |_||_|    |___||_| \_|
+@项目名称 : python-main
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+)
+    multiprocessing.freeze_support()  # 使用pyinstaller打包后，如果不加这行代码，并行处理会出问题
+    argv = sys.argv
+    env = 'dev'
+    if len(argv) > 1:
+        env = argv[1]
+
+    cf = config.Config(res_dir + '/conf.ini', 'UTF-8', env)
+    config.add_conf('env', env)
+    config.add_conf('root', workspace)
+    host = config.MAP['server_host'] if config.MAP['server_host'] != '0.0.0.0' else http.get_local_ip()
+    port = int(config.MAP['server_port'])
+    logger.Logger()
+    from database import db_handler as dh
+    dh._init_lib()
+    dh._init_local_db()
+    logger.debug(f"初始化oracle lib")
+    logger.debug(f"----- 工作路径：{workspace} -----")
+    logger.debug(f"----- 服务器启动，连接地址 http://{host}:{port} -----")
+    logger.debug(f"----- Swagger UI，连接地址 http://{host}:{port}/docs -----")
+
+    from api import etl
+
+    app.include_router(etl.router, prefix="/etl", tags=["ETL工具"])
+
+    uvicorn.run(app, host=host, port=port)

database_smartools/module/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+@项目名称 : yhfin-data-agent
+@文件名称 : __init__.py.py
+@创建人   : zhongbinjie
+@创建时间 : 2025/9/9 17:02
+@文件说明 : 
+@企业名称 : 深圳市赢和信息技术有限公司
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+
+
+def test():
+    print()
+
+
+if __name__ == '__main__':
+    None

database_smartools/module/function.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+@项目名称 : python-main
+@文件名称 : function.py
+@创建人   : zhongbinjie
+@创建时间 : 2025/6/7 19:06
+@文件说明 : 
+@企业名称 : 深圳市赢和信息技术有限公司
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+
+import importlib
+import os.path
+import sys
+from pathlib import Path
+from utils import logger, config, file, timer
+
+
+def execute_function(func_name, args=None, script_dir=None):
+    """
+    通过函数名动态执行对应脚本中的函数
+
+    :param func_name: 要执行的函数名称
+    :param args: 函数传入的参数
+    :param script_dir: 存放函数脚本的目录，默认为conf.ini -> common -> script_dir
+    """
+    result = {
+        'status': False,
+        'data': {},
+        'message': f"函数{func_name}运行时发生错误，未知异常",
+        'time': 0
+    }
+    # 获取函数脚本目录的绝对路径
+    base_dir = config.MAP['root']  # 当前项目所在目录
+    target_dir = os.path.join(base_dir, config.MAP['script_dir'])
+    if script_dir:
+        target_dir = script_dir
+    # 临时添加目标目录到sys.path
+    original_sys_path = sys.path.copy()
+    start_time = timer.get_time()
+    try:
+
+        root, filename = file.search_script(target_dir, func_name + '.py')
+        if not root:
+            end_time = timer.get_time()
+            message = f"错误：找不到名为'{func_name}'的脚本。"
+            logger.error(message)
+            result['status'] = False
+            result['message'] = message
+            result['time'] = timer.get_timediff(end_time, start_time)
+            return result
+
+        sys.path.insert(0, str(root))
+
+        try:
+            # 动态导入模块
+            module = importlib.import_module(func_name)
+
+            # 获取函数对象
+            func = getattr(module, func_name)
+            logger.debug(f"获取函数对象，module: {module}, func_name: {func_name}")
+
+            # 执行函数
+            rtn_map = None
+            logger.debug(f"运行函数，func_name: {func_name}")
+            if args:
+                rtn_map = func(args)
+            else:
+                rtn_map = func()
+            end_time = timer.get_time()
+            dur_time = timer.get_timediff(end_time, start_time)
+            result['time'] = dur_time
+            logger.debug(f"函数运行结束，运行时间: {dur_time}")
+            if rtn_map is not None:
+                result['status'] = rtn_map['result']
+                if 'data' in rtn_map.keys():
+                    result['data'] = rtn_map['data']
+                result['message'] = rtn_map['message']
+
+        except ImportError as e:
+            end_time = timer.get_time()
+            message = f"脚本'{func_name}'导入异常。错误信息：{e}"
+            logger.error(message)
+            result['status'] = False
+            result['message'] = message
+            result['time'] = timer.get_timediff(end_time, start_time)
+
+        except AttributeError as e:
+            end_time = timer.get_time()
+            message = f"函数{func_name}运行时发生错误，错误信息：{e}。"
+            logger.error(message)
+            result['status'] = False
+            result['message'] = message
+            result['time'] = timer.get_timediff(end_time, start_time)
+
+        finally:
+            # 从 sys.modules 中移除已导入的模块
+            if func_name in sys.modules:
+                del sys.modules[func_name]
+            # 从 sys.path 中移除临时添加的路径
+            if str(root) in sys.path:
+                sys.path.remove(str(root))
+
+    except Exception as e:
+        end_time = timer.get_time()
+        message = f"函数{func_name}运行时发生错误，错误信息：{e}。"
+        logger.error(message)
+        result['status'] = False
+        result['message'] = message
+        result['time'] = timer.get_timediff(end_time, start_time)
+
+    finally:
+        # 恢复原始sys.path
+        sys.path = original_sys_path
+        return result

database_smartools/utils/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+@项目名称 : yhfin-data-agent
+@文件名称 : __init__.py.py
+@创建人   : zhongbinjie
+@创建时间 : 2025/9/9 17:02
+@文件说明 : 
+@企业名称 : 深圳市赢和信息技术有限公司
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+
+
+def test():
+    print()
+
+
+if __name__ == '__main__':
+    None

database_smartools/utils/config.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""
+@项目名称 : python-main
+@文件名称 : config.py
+@创建人   : zhongbinjie
+@创建时间 : 2025/6/7 19:06
+@文件说明 : 
+@企业名称 : 深圳市赢和信息技术有限公司
+@Copyright:2025-2035, 深圳市赢和信息技术有限公司. All rights Reserved.
+"""
+
+import configparser
+
+MAP = {}
+
+def add_conf(key, value):
+    global MAP
+    MAP[key] = value
+
+class Config(object):
+    env : str
+    def __init__(self, filename, encoding, env):
+        print("初始化配置信息")
+        global MAP
+        MAP = {}
+        # 声明配置类对象
+        self.config = configparser.ConfigParser()
+        # 读取配置文件
+        self.config.read(filename, encoding)
+
+        try:
+            items = self.get_items('common')
+            for key, conf in items:
+                setattr(self, key, conf)
+
+            items = self.get_items(env)
+            for key, conf in items:
+                setattr(self, key, conf)
+                if key == 'biz_db_type':
+                    db_confs = self.get_items(conf)
+                    for key, conf in db_confs:
+                        setattr(self, key, conf)
+
+            MAP = self.__dict__
+            # print("配置对象属性", MAP)
+        except Exception as e:
+            print(e)
+
+    def get_options(self, section):
+        """获取 option"""
+        return self.config.options(section)
+
+    def get_items(self, section):
+        """获取 items"""
+        return self.config.items(section)
+