pixelarraylib 1.0.0__py3-none-any.whl

arraylib/aliyun/sts.py

@@ -0,0 +1,124 @@
+import traceback
+import json
+import oss2
+from alibabacloud_sts20150401.client import Client as StsClient
+from alibabacloud_tea_openapi import models as open_api_models
+from alibabacloud_sts20150401 import models as sts_20150401_models
+from alibabacloud_tea_util import models as util_models
+from arraylib.monitor.feishu import Feishu
+from arraylib.db_utils.redis import RedisUtils
+
+feishu_alert = Feishu("devtoolkit服务报警")
+
+
+class STSUtils:
+    def __init__(
+        self, access_key_id, access_key_secret, role_arn, region_id, bucket_name, redis_utils
+    ):
+        assert isinstance(redis_utils, RedisUtils), "redis_utils must be a RedisUtils instance"
+        self.access_key_id = access_key_id
+        self.access_key_secret = access_key_secret
+        self.role_arn = role_arn
+        # 使用新版本的STS客户端
+        config = open_api_models.Config(
+            access_key_id=access_key_id,
+            access_key_secret=access_key_secret,
+            region_id=region_id,
+        )
+        self.client = StsClient(config)
+        self.redis_utils = redis_utils
+        self.bucket_name = bucket_name
+        self.oss_endpoint = f"https://oss-{region_id}.aliyuncs.com"
+        self.region_id = region_id
+
+    def get_sts_token(self, role_session_name, duration_seconds=3600):
+        """
+        description:
+            获取STS临时访问凭证
+        parameters:
+            role_session_name(str): 角色会话名称
+            duration_seconds(int): 临时凭证有效期，单位为秒
+        return:
+            credentials(dict): 临时访问凭证
+            flag(bool): 是否成功
+        """
+        assert role_session_name in ["oss-session"]
+        try:
+            credentials = self.redis_utils.get(f"sts_auth_token_{role_session_name}")
+            if credentials:
+                return json.loads(credentials), True
+
+            assume_role_request = sts_20150401_models.AssumeRoleRequest(
+                role_arn=self.role_arn,
+                role_session_name=role_session_name,
+                duration_seconds=duration_seconds,
+            )
+
+            response = self.client.assume_role_with_options(
+                assume_role_request, util_models.RuntimeOptions()
+            )
+
+            credentials = response.to_map()["body"]["Credentials"]
+
+            access_key_id = credentials["AccessKeyId"]
+            access_key_secret = credentials["AccessKeySecret"]
+            security_token = credentials["SecurityToken"]
+
+            credentials = {
+                "access_key_id": access_key_id,
+                "access_key_secret": access_key_secret,
+                "security_token": security_token,
+            }
+            self.redis_utils.set(
+                f"sts_auth_token_{role_session_name}",
+                json.dumps(credentials),
+                expire_seconds=duration_seconds,
+            )
+            return credentials, True
+        except Exception as e:
+            feishu_alert.send(traceback.format_exc())
+            return {}, False
+
+    def get_oss_sts_client(self, endpoint, bucket_name):
+        """
+        description:
+            获取基于STS的OSS客户端
+        return:
+            bucket(oss2.Bucket): OSS客户端
+            flag(bool): 是否成功
+        """
+        try:
+            credentials, flag = self.get_sts_token("oss-session")
+            if not credentials or not flag:
+                return None, False
+
+            auth = oss2.StsAuth(
+                credentials["access_key_id"],
+                credentials["access_key_secret"],
+                credentials["security_token"],
+            )
+            return oss2.Bucket(auth, endpoint, bucket_name), True
+        except Exception as e:
+            feishu_alert.send(traceback.format_exc())
+            return None, False
+
+    def generate_presigned_url(self, prefix, expires_in=60 * 60 * 24):
+        """
+        description:
+            使用STS生成预签名URL
+        parameters:
+            prefix(str): 前缀
+            expires_in(int): 过期时间，默认24小时
+        return:
+            url(str): 预签名URL
+        """
+        try:
+            sts_oss_client, sts_flag = self.get_oss_sts_client(
+                self.oss_endpoint, self.bucket_name
+            )
+            if not sts_oss_client or not sts_flag:
+                return ""
+            return sts_oss_client.sign_url("GET", prefix, expires_in)
+        except Exception as e:
+            feishu_alert.send(traceback.format_exc())
+            return ""

arraylib/db_utils/mysql.py

@@ -0,0 +1,544 @@
+import traceback
+import pymysql
+import aiomysql
+import asyncio
+from arraylib.monitor.feishu import Feishu
+import time
+from pymysql.err import OperationalError
+
+feishu_alert = Feishu("devtoolkit服务报警")
+
+
+class MysqlUtils:
+    def __init__(
+        self,
+        host,
+        database,
+        user,
+        password,
+        port,
+        max_retries=3,
+    ):
+        self.host = host
+        self.database = database
+        self.user = user
+        self.password = password
+        self.port = port
+        self.max_retries = max_retries
+        self.mysql = None
+        self._connect()
+
+    def _connect(self):
+        """建立数据库连接，支持重试"""
+        for attempt in range(self.max_retries):
+            try:
+                self.mysql = pymysql.connect(
+                    host=self.host,
+                    database=self.database,
+                    user=self.user,
+                    password=self.password,
+                    port=self.port,
+                    connect_timeout=60,
+                    read_timeout=60,
+                    write_timeout=60,
+                    autocommit=True,
+                )
+                break  # 连接成功，跳出重试循环
+            except OperationalError as e:
+                if attempt < self.max_retries - 1:
+                    feishu_alert.send(
+                        f"MySQL连接失败，正在重试 ({attempt + 1}/{self.max_retries}): {str(e)}"
+                    )
+                    time.sleep(2**attempt)  # 指数退避
+                else:
+                    feishu_alert.send(f"MySQL连接最终失败: {traceback.format_exc()}")
+                    raise
+
+    def get_conn(self):
+        return self.mysql
+
+    def _ensure_connection(self):
+        """确保连接有效，如果断开则重新连接"""
+        try:
+            self.mysql.ping(reconnect=True)
+        except Exception:
+            self._connect()
+
+    def get_db_name(self):
+        """
+        description:
+            获取当前数据库名称
+        return:
+            database_name(str): 数据库名称
+        """
+        res = self.query("SELECT DATABASE();")
+        database_name = res[0][0]
+        return database_name
+
+    def create_table(self, table_name, columns):
+        """
+        创建表
+        parameters:
+            table_name(str): 表名
+            columns(list(tuple)): 列名和类型
+        returnType:
+            flag(bool): 是否成功
+        """
+        sql = f"""
+            CREATE TABLE {table_name} (
+                {','.join([f'{col_name} {type}' for col_name, type in columns])}
+            );
+        """
+        return self.execute_sql(sql)
+
+    def add_column(self, table_name, column_name, column_type):
+        """
+        description:
+            添加列
+        parameters:
+            table_name(str): 表名
+            column_name(str): 列名
+            column_type(str): 列类型
+        return:
+            flag(bool): 是否成功
+        """
+        sql = f"""
+            ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type};
+        """
+        return self.execute_sql(sql)
+
+    def delete_column(self, table_name, column_name):
+        """
+        description:
+            删除列
+        parameters:
+            table_name(str): 表名
+            column_name(str): 列名
+        return:
+            flag(bool): 是否成功
+        """
+        sql = f"""
+            ALTER TABLE {table_name} DROP COLUMN {column_name};
+        """
+        return self.execute_sql(sql)
+
+    def get_table_names(self):
+        """
+        description:
+            获取当前数据库中的所有表名
+        return:
+            table_names(list): 表名列表
+        """
+        res = self.query("SHOW TABLES;")
+        table_names = [row[0] for row in res]
+        return table_names
+
+    def insert(self, table_name, rows, batch_size=100):
+        """
+        description:
+            插入数据
+        parameters:
+            table_name(str): 表名
+            rows(list(dict)): 数据
+            batch_size(int): 每批插入的条数
+        return:
+            flag(bool): 是否成功
+        """
+        for i in range(0, len(rows), batch_size):
+            batch_rows = rows[i : i + batch_size]
+            values_str_list = []
+            for row in batch_rows:
+                values_str = ",".join([f"'{value}'" for value in row.values()])
+                values_str = f"({values_str})"
+                values_str_list.append(values_str)
+            sql = f"""
+                INSERT INTO {table_name} ({','.join(rows[0].keys())})
+                VALUES
+                {','.join(values_str_list)};
+            """
+            self.execute_sql(sql)
+            print(f"insert {i} / {len(rows)} rows to {table_name}")
+        return True
+
+    def insert_or_update(self, table_name, rows, key_columns, batch_size=100):
+        """
+        description:
+            插入数据，如果指定的键已存在则更新
+        parameters:
+            table_name(str): 表名
+            rows(list(dict)): 数据
+            key_columns(list): 作为唯一标识的列名列表
+            batch_size(int): 每批插入的条数
+        return:
+            flag(bool): 是否成功
+        """
+        try:
+            for i in range(0, len(rows), batch_size):
+                batch_rows = rows[i : i + batch_size]
+                values_str_list = []
+                for row in batch_rows:
+                    values_str = ",".join([f"'{value}'" for value in row.values()])
+                    values_str = f"({values_str})"
+                    values_str_list.append(values_str)
+
+                update_str = ",".join(
+                    [f"{k}=VALUES({k})" for k in rows[0].keys() if k not in key_columns]
+                )
+                on_duplicate_key = (
+                    f"ON DUPLICATE KEY UPDATE {update_str}" if update_str else ""
+                )
+
+                sql = f"""
+                    INSERT INTO {table_name} ({','.join(rows[0].keys())})
+                    VALUES
+                    {','.join(values_str_list)}
+                    {on_duplicate_key};
+                """
+                self.execute_sql(sql)
+                print(f"插入/更新 {i} / {len(rows)} 行到 {table_name}")
+            return True
+        except Exception as e:
+            feishu_alert.send(f"mysql-insert_or_update error {traceback.format_exc()}")
+            return False
+
+    def query(self, sql, convert_to_dict=False):
+        """
+        description:
+            执行sql查询语句
+        parameters:
+            sql(str): sql语句
+            convert_to_dict(bool): 是否将查询结果转换为字典
+        return:
+            result(list): 查询结果
+        """
+        allowed_sql_prefixes = ("SELECT", "SHOW", "DESCRIBE", "EXPLAIN")
+        assert (
+            sql.lstrip().upper().startswith(allowed_sql_prefixes)
+        ), f"sql 必须以 {', '.join(allowed_sql_prefixes)} 开头"
+        try:
+            self._ensure_connection()  # 确保连接有效
+            cursor = self.mysql.cursor()
+            cursor.execute(sql)
+            result = cursor.fetchall()
+            columns = [col[0] for col in cursor.description]
+            if convert_to_dict:
+                return [dict(zip(columns, row)) for row in result]
+            return list(result)
+        except Exception as e:
+            feishu_alert.send(f"mysql-query error {traceback.format_exc()}")
+            return []
+        finally:
+            cursor.close()
+
+    def execute_sql(self, sql):
+        """
+        description:
+            执行sql语句
+        parameters:
+            sql(str): sql语句
+        return:
+            flag(bool): 是否成功
+        """
+        try:
+            self._ensure_connection()  # 确保连接有效
+            cursor = self.mysql.cursor()
+            cursor.execute(sql)
+            self.mysql.commit()
+            return True
+        except Exception as e:
+            self.mysql.rollback()
+            feishu_alert.send(f"mysql-execute error {traceback.format_exc()}")
+            return False
+        finally:
+            cursor.close()
+
+    def clear_table(self, table_name):
+        sql = f"""
+            TRUNCATE TABLE {table_name};
+        """
+        return self.execute_sql(sql)
+
+    def delete_table(self, table_name):
+        sql = f"""
+            DROP TABLE {table_name};
+        """
+        return self.execute_sql(sql)
+
+
+class MysqlUtilsAsync:
+    def __init__(
+        self,
+        host,
+        database,
+        user,
+        password,
+        port,
+        max_retries=3,
+    ):
+        self.host = host
+        self.database = database
+        self.user = user
+        self.password = password
+        self.port = port
+        self.max_retries = max_retries
+        self.async_mysql = None
+
+    async def get_async_conn(self):
+        """获取异步MySQL连接"""
+        if self.async_mysql is None:
+            await self._async_connect()
+        return self.async_mysql
+
+    async def _async_connect(self):
+        """建立异步数据库连接，支持重试"""
+        for attempt in range(self.max_retries):
+            try:
+                self.async_mysql = await aiomysql.connect(
+                    host=self.host,
+                    db=self.database,
+                    user=self.user,
+                    password=self.password,
+                    port=self.port,
+                    connect_timeout=60,
+                    autocommit=True,
+                )
+                break  # 连接成功，跳出重试循环
+            except Exception as e:
+                if attempt < self.max_retries - 1:
+                    feishu_alert.send(
+                        f"异步MySQL连接失败，正在重试 ({attempt + 1}/{self.max_retries}): {str(e)}"
+                    )
+                    await asyncio.sleep(2**attempt)  # 指数退避
+                else:
+                    feishu_alert.send(
+                        f"异步MySQL连接最终失败: {traceback.format_exc()}"
+                    )
+                    raise
+
+    async def _async_ensure_connection(self):
+        """确保异步连接有效，如果断开则重新连接"""
+        try:
+            if self.async_mysql is None:
+                await self._async_connect()
+            else:
+                # aiomysql没有ping方法，直接尝试执行简单查询
+                cursor = await self.async_mysql.cursor()
+                await cursor.execute("SELECT 1")
+                await cursor.close()
+        except Exception:
+            await self._async_connect()
+
+    async def get_db_name(self):
+        """
+        description:
+            异步获取当前数据库名称
+        return:
+            database_name(str): 数据库名称
+        """
+        res = await self.query("SELECT DATABASE();")
+        database_name = res[0][0]
+        return database_name
+
+    async def create_table(self, table_name, columns):
+        """
+        异步创建表
+        parameters:
+            table_name(str): 表名
+            columns(list(tuple)): 列名和类型
+        returnType:
+            flag(bool): 是否成功
+        """
+        sql = f"""
+            CREATE TABLE {table_name} (
+                {','.join([f'{col_name} {type}' for col_name, type in columns])}
+            );
+        """
+        return await self.execute_sql(sql)
+
+    async def add_column(self, table_name, column_name, column_type):
+        """
+        description:
+            异步添加列
+        parameters:
+            table_name(str): 表名
+            column_name(str): 列名
+            column_type(str): 列类型
+        return:
+            flag(bool): 是否成功
+        """
+        sql = f"""
+            ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type};
+        """
+        return await self.execute_sql(sql)
+
+    async def delete_column(self, table_name, column_name):
+        """
+        description:
+            异步删除列
+        parameters:
+            table_name(str): 表名
+            column_name(str): 列名
+        return:
+            flag(bool): 是否成功
+        """
+        sql = f"""
+            ALTER TABLE {table_name} DROP COLUMN {column_name};
+        """
+        return await self.execute_sql(sql)
+
+    async def get_table_names(self):
+        """
+        description:
+            异步获取当前数据库中的所有表名
+        return:
+            table_names(list): 表名列表
+        """
+        res = await self.query("SHOW TABLES;")
+        table_names = [row[0] for row in res]
+        return table_names
+
+    async def insert(self, table_name, rows, batch_size=100):
+        """
+        description:
+            异步插入数据
+        parameters:
+            table_name(str): 表名
+            rows(list(dict)): 数据
+            batch_size(int): 每批插入的条数
+        return:
+            flag(bool): 是否成功
+        """
+        for i in range(0, len(rows), batch_size):
+            batch_rows = rows[i : i + batch_size]
+            values_str_list = []
+            for row in batch_rows:
+                values_str = ",".join([f"'{value}'" for value in row.values()])
+                values_str = f"({values_str})"
+                values_str_list.append(values_str)
+            sql = f"""
+                INSERT INTO {table_name} ({','.join(rows[0].keys())})
+                VALUES
+                {','.join(values_str_list)};
+            """
+            await self.execute_sql(sql)
+            print(f"insert {i} / {len(rows)} rows to {table_name}")
+        return True
+
+    async def insert_or_update(self, table_name, rows, key_columns, batch_size=100):
+        """
+        description:
+            异步插入数据，如果指定的键已存在则更新
+        parameters:
+            table_name(str): 表名
+            rows(list(dict)): 数据
+            key_columns(list): 作为唯一标识的列名列表
+            batch_size(int): 每批插入的条数
+        return:
+            flag(bool): 是否成功
+        """
+        try:
+            for i in range(0, len(rows), batch_size):
+                batch_rows = rows[i : i + batch_size]
+                values_str_list = []
+                for row in batch_rows:
+                    values_str = ",".join([f"'{value}'" for value in row.values()])
+                    values_str = f"({values_str})"
+                    values_str_list.append(values_str)
+
+                update_str = ",".join(
+                    [f"{k}=VALUES({k})" for k in rows[0].keys() if k not in key_columns]
+                )
+                on_duplicate_key = (
+                    f"ON DUPLICATE KEY UPDATE {update_str}" if update_str else ""
+                )
+
+                sql = f"""
+                    INSERT INTO {table_name} ({','.join(rows[0].keys())})
+                    VALUES
+                    {','.join(values_str_list)}
+                    {on_duplicate_key};
+                """
+                await self.execute_sql(sql)
+                print(f"插入/更新 {i} / {len(rows)} 行到 {table_name}")
+            return True
+        except Exception as e:
+            feishu_alert.send(
+                f"mysql-insert_or_update_async error {traceback.format_exc()}"
+            )
+            return False
+
+    async def query(self, sql, convert_to_dict=False):
+        """
+        description:
+            异步执行sql查询语句
+        parameters:
+            sql(str): sql语句
+            convert_to_dict(bool): 是否将查询结果转换为字典
+        return:
+            result(list): 查询结果
+        """
+        allowed_sql_prefixes = ("SELECT", "SHOW", "DESCRIBE", "EXPLAIN")
+        assert (
+            sql.lstrip().upper().startswith(allowed_sql_prefixes)
+        ), f"sql 必须以 {', '.join(allowed_sql_prefixes)} 开头"
+        cursor = None
+        try:
+            await self._async_ensure_connection()  # 确保连接有效
+            async_mysql = await self.get_async_conn()
+            cursor = await async_mysql.cursor()
+            await cursor.execute(sql)
+            result = await cursor.fetchall()
+            columns = [col[0] for col in cursor.description]
+            if convert_to_dict:
+                return [dict(zip(columns, row)) for row in result]
+            return list(result)
+        except Exception as e:
+            feishu_alert.send(f"mysql-query_async error {traceback.format_exc()}")
+            return []
+        finally:
+            if cursor:
+                await cursor.close()
+
+    async def execute_sql(self, sql):
+        """
+        description:
+            异步执行sql语句
+        parameters:
+            sql(str): sql语句
+        return:
+            flag(bool): 是否成功
+        """
+        cursor = None
+        try:
+            await self._async_ensure_connection()  # 确保连接有效
+            async_mysql = await self.get_async_conn()
+            cursor = await async_mysql.cursor()
+            await cursor.execute(sql)
+            await async_mysql.commit()
+            return True
+        except Exception as e:
+            async_mysql = await self.get_async_conn()
+            await async_mysql.rollback()
+            feishu_alert.send(f"mysql-execute_async error {traceback.format_exc()}")
+            return False
+        finally:
+            if cursor:
+                await cursor.close()
+
+    async def clear_table(self, table_name):
+        sql = f"""
+            TRUNCATE TABLE {table_name};
+        """
+        return await self.execute_sql(sql)
+
+    async def delete_table(self, table_name):
+        sql = f"""
+            DROP TABLE {table_name};
+        """
+        return await self.execute_sql(sql)
+
+    async def close(self):
+        """关闭异步MySQL连接"""
+        if self.async_mysql:
+            self.async_mysql.close()
+            await self.async_mysql.wait_closed()