re-common 10.0.39__py3-none-any.whl → 10.0.41__py3-none-any.whl

re_common/v2/baselibrary/utils/db.py

@@ -1,156 +1,174 @@
-import os
-import time
-
-import aiomysql
-import asyncio
-from contextlib import asynccontextmanager
-from typing import AsyncGenerator, Tuple
-from collections import namedtuple
-
-from aiomysql import Pool, Connection, Cursor
-
-DB_CONFIG = {
-    "host": "192.168.98.64",
-    "port": 4000,
-    "user": "dataware_house_baseUser",
-    "password": "FF19AF831AEBD580B450B16BF9264200",
-    "db": "dataware_house_base",
-    "charset": "utf8mb4",
-    "minsize": 16,  # 最小连接数
-    "maxsize": 128,  # 最大连接数
-    "autocommit": False,  # 自动提交事务
-    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    "echo": False,  # 打印SQL语句
-}
-
-DB_CONFIG1 = {
-    "host": "192.168.98.64",
-    "port": 4000,
-    "user": "foreign_fulltextUser",
-    "password": "i4hIeasw1qpmhGN2nwL7",
-    "db": "foreign_fulltext",
-    "charset": "utf8mb4",
-    "minsize": 16,  # 最小连接数
-    "maxsize": 128,  # 最大连接数
-    "autocommit": False,  # 自动提交事务
-    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    "echo": False,  # 打印SQL语句
-}
-
-
-async def get_pool_only(_DB_CONFIG: dict = None):
-    global DB_CONFIG
-    if _DB_CONFIG is not None:
-        DB_CONFIG = _DB_CONFIG
-    pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
-    return pool
-
-
-@asynccontextmanager
-async def get_db_pool(_DB_CONFIG: dict = None):
-    """异步数据库连接池管理工具"""
-    global DB_CONFIG
-    if _DB_CONFIG is not None:
-        DB_CONFIG = _DB_CONFIG
-    pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
-    try:
-        yield pool
-    finally:
-        pool.close()
-        await pool.wait_closed()
-
-
-@asynccontextmanager
-async def get_session(pool: Pool) -> AsyncGenerator[Tuple[Connection, Cursor], None]:
-    """获取数据库会话"""
-    async with pool.acquire() as conn:
-        async with conn.cursor() as cursor:
-            yield conn, cursor
-
-
-async def dictfetchall(cursor: Cursor):
-    """
-    Return all rows from a cursor as a dict.
-    Assume the column names are unique.
-    """
-    columns = [col[0] for col in cursor.description]
-    return [dict(zip(columns, row)) for row in await cursor.fetchall()]
-
-
-async def namedtuplefetchall(cursor: Cursor):
-    """
-    Return all rows from a cursor as a namedtuple.
-    Assume the column names are unique.
-    """
-    desc = cursor.description
-    nt_result = namedtuple("Result", [col[0] for col in desc])
-    return [nt_result(*row) for row in await cursor.fetchall()]
-
-
-# main.py
-
-
-aiomysql_pool = None
-pool_lock = asyncio.Lock()  # 全局异步锁
-
-
-
-async def init_aiomysql_pool_async():
-    global aiomysql_pool
-    if aiomysql_pool is None:
-        async with pool_lock:
-            if aiomysql_pool is None:
-                print(f"[{os.getpid()}] Initializing aiomysql pool...")
-                aiomysql_pool = await aiomysql.create_pool(**DB_CONFIG)
-    return aiomysql_pool
-
-
-motor_fs = None
-client = None
-motor_fs_lock = asyncio.Lock()  # 全局异步锁
-_loop_id_mongo = None
-
-async def check_connection(client):
-    try:
-        print("check mongodb client ping")
-        await client.admin.command("ping")
-        return True
-    except Exception:
-        return False
-
-
-async def init_motor_async(uri, db_name, bucket_name, is_reload=False):
-    global motor_fs, client, _loop_id_mongo
-    is_ping = True
-
-    if _loop_id_mongo is not None:
-        loop_id = id(asyncio.get_running_loop())
-        if loop_id != _loop_id_mongo:
-            is_reload = True
-
-    # 防止 每次都检查 只有 is_reload 时才检查连接
-    if is_reload:
-        is_ping = await check_connection(client)
-    if motor_fs is None or not is_ping:
-        async with motor_fs_lock:
-            if motor_fs is None or not is_ping:
-                print(f"[{os.getpid()}] Initializing motor_fs...")
-                from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
-                client = AsyncIOMotorClient(uri)
-                db = client[db_name]
-                motor_fs = AsyncIOMotorGridFSBucket(database=db, bucket_name=bucket_name)
-                _loop_id_mongo = id(asyncio.get_running_loop())
-    return motor_fs, client
-
-# async def run_main():
-#     while True:
-#         uri = "mongodb://192.168.98.80:27001/wpdc"
-#         db_name = "wpdc"
-#         bucket_name = "sci_doc"
-#         motor_fs, client = await init_motor_async(uri, db_name, bucket_name,is_reload=True)
-#         # print(await check_connection(client))
-#         time.sleep(3)
-#
-#
-# if __name__ == "__main__":
-#     asyncio.run(run_main())
+import os
+import time
+
+import aiomysql
+import asyncio
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator, Tuple
+from collections import namedtuple
+
+from aiomysql import Pool, Connection, Cursor
+
+DB_CONFIG = {
+    "host": "192.168.98.64",
+    "port": 4000,
+    "user": "dataware_house_baseUser",
+    "password": "FF19AF831AEBD580B450B16BF9264200",
+    "db": "dataware_house_base",
+    "charset": "utf8mb4",
+    "minsize": 16,  # 最小连接数
+    "maxsize": 128,  # 最大连接数
+    "autocommit": False,  # 自动提交事务
+    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    "echo": False,  # 打印SQL语句
+}
+
+DB_CONFIG1 = {
+    "host": "192.168.98.64",
+    "port": 4000,
+    "user": "foreign_fulltextUser",
+    "password": "i4hIeasw1qpmhGN2nwL7",
+    "db": "foreign_fulltext",
+    "charset": "utf8mb4",
+    "minsize": 16,  # 最小连接数
+    "maxsize": 128,  # 最大连接数
+    "autocommit": False,  # 自动提交事务
+    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    "echo": False,  # 打印SQL语句
+}
+
+
+async def get_pool_only(_DB_CONFIG: dict = None):
+    global DB_CONFIG
+    if _DB_CONFIG is not None:
+        DB_CONFIG = _DB_CONFIG
+    pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
+    return pool
+
+
+@asynccontextmanager
+async def get_db_pool(_DB_CONFIG: dict = None):
+    """异步数据库连接池管理工具"""
+    global DB_CONFIG
+    if _DB_CONFIG is not None:
+        DB_CONFIG = _DB_CONFIG
+    pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
+    try:
+        yield pool
+    finally:
+        pool.close()
+        await pool.wait_closed()
+
+
+@asynccontextmanager
+async def get_session(pool: Pool) -> AsyncGenerator[Tuple[Connection, Cursor], None]:
+    """获取数据库会话"""
+    async with pool.acquire() as conn:
+        async with conn.cursor() as cursor:
+            yield conn, cursor
+
+
+async def dictfetchall(cursor: Cursor):
+    """
+    Return all rows from a cursor as a dict.
+    Assume the column names are unique.
+    """
+    columns = [col[0] for col in cursor.description]
+    return [dict(zip(columns, row)) for row in await cursor.fetchall()]
+
+
+async def namedtuplefetchall(cursor: Cursor):
+    """
+    Return all rows from a cursor as a namedtuple.
+    Assume the column names are unique.
+    """
+    desc = cursor.description
+    nt_result = namedtuple("Result", [col[0] for col in desc])
+    return [nt_result(*row) for row in await cursor.fetchall()]
+
+
+# main.py
+
+
+aiomysql_pool = None
+pool_lock = asyncio.Lock()  # 全局异步锁
+
+
+async def init_aiomysql_pool_async():
+    global aiomysql_pool
+    if aiomysql_pool is None:
+        async with pool_lock:
+            if aiomysql_pool is None:
+                print(f"[{os.getpid()}] Initializing aiomysql pool...")
+                aiomysql_pool = await aiomysql.create_pool(**DB_CONFIG)
+    return aiomysql_pool
+
+
+motor_fs = None
+client = None
+motor_fs_lock = asyncio.Lock()  # 全局异步锁
+_loop_id_mongo = None
+
+
+async def check_connection(client):
+    try:
+        print("check mongodb client ping")
+        await client.admin.command("ping")
+        return True
+    except Exception:
+        return False
+
+
+async def init_motor_async(uri, db_name, bucket_name, is_reload=False):
+    global motor_fs, client, _loop_id_mongo
+    is_ping = True
+
+    if _loop_id_mongo is not None:
+        loop_id = id(asyncio.get_running_loop())
+        if loop_id != _loop_id_mongo:
+            is_reload = True
+
+    # 防止 每次都检查 只有 is_reload 时才检查连接
+    if is_reload:
+        is_ping = await check_connection(client)
+    if motor_fs is None or not is_ping:
+        async with motor_fs_lock:
+            if motor_fs is None or not is_ping:
+                print(f"[{os.getpid()}] Initializing motor_fs...")
+                from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
+                client = AsyncIOMotorClient(uri)
+                db = client[db_name]
+                motor_fs = AsyncIOMotorGridFSBucket(database=db, bucket_name=bucket_name)
+                _loop_id_mongo = id(asyncio.get_running_loop())
+    return motor_fs, client
+
+
+# async def run_main():
+#     while True:
+#         uri = "mongodb://192.168.98.80:27001/wpdc"
+#         db_name = "wpdc"
+#         bucket_name = "sci_doc"
+#         motor_fs, client = await init_motor_async(uri, db_name, bucket_name,is_reload=True)
+#         # print(await check_connection(client))
+#         time.sleep(3)
+#
+#
+# if __name__ == "__main__":
+#     asyncio.run(run_main())
+
+
+def get_connection(autocommit: bool = True) -> Connection:
+    from pymysql import Connection
+    from pymysql.cursors import DictCursor
+    import pymysql
+    db_conf = {
+        "host": "192.168.98.55",
+        "port": 4000,
+        "user": "dataware_house_baseUser",
+        "password": "FF19AF831AEBD580B450B16BF9264200",
+        "database": "dataware_house_base",
+        "autocommit": autocommit,
+        "cursorclass": DictCursor,
+    }
+    conn: Connection = pymysql.connect(**db_conf)
+    return conn

re_common/v2/baselibrary/utils/elasticsearch.py

@@ -0,0 +1,46 @@
+# elasticsearch[async]~=8.11.1
+from typing import Any, Dict, List
+from elasticsearch import AsyncElasticsearch, helpers
+
+DEFAULT_CONFIG = {
+    "hosts": [
+        "http://192.168.32.97:9200",
+        "http://192.168.32.99:9200",
+        "http://192.168.32.101:9200",
+        "http://192.168.32.103:9200",
+    ],
+    "basic_auth": ("elastic", "Zl2bWsOuvqi0IUwpvGhK"),
+    "timeout": 60,
+}
+
+
+def get_es(config=DEFAULT_CONFIG):
+    es = AsyncElasticsearch(**config)
+    return es
+
+
+async def update(es: AsyncElasticsearch, index: str, doc_id: str, doc: Dict[str, Any], doc_as_upsert: bool = False):
+    return await es.update(index=index, id=doc_id, doc=doc, doc_as_upsert=doc_as_upsert)
+
+
+async def bulk_update(es: AsyncElasticsearch, index: str, docs: List[Dict[str, Any]], doc_as_upsert: bool = False):
+    """
+    批量更新ES文档
+
+    docs 格式示例：
+    [
+        {"_id": "1", "doc": {"field1": "value1"}},
+        {"_id": "2", "doc": {"field2": "value2"}},
+    ]
+    """
+    actions = []
+    for item in docs:
+        action = {
+            "_op_type": "update",
+            "_index": index,
+            "_id": item["_id"],
+            "doc": item["doc"],
+            "doc_as_upsert": doc_as_upsert,
+        }
+        actions.append(action)
+    return await helpers.async_bulk(es, actions=actions)

re_common/v2/baselibrary/utils/json_cls.py

@@ -1,16 +1,16 @@
-import base64
-import functools
-import json
-
-json_dumps = functools.partial(json.dumps, ensure_ascii=False)
-
-
-class BytesEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, bytes):
-            return base64.b64encode(obj).decode('utf-8')
-        return super().default(obj)
-
-
-def base64_to_bytes(base64_str, encoding="utf-8") -> bytes:
-    return base64.b64decode(base64_str.encode(encoding))
+import base64
+import functools
+import json
+
+json_dumps = functools.partial(json.dumps, ensure_ascii=False)
+
+
+class BytesEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, bytes):
+            return base64.b64encode(obj).decode('utf-8')
+        return super().default(obj)
+
+
+def base64_to_bytes(base64_str, encoding="utf-8") -> bytes:
+    return base64.b64decode(base64_str.encode(encoding))

re_common/v2/baselibrary/utils/mq.py

@@ -1,83 +1,83 @@
-import logging
-import traceback
-
-from re_common.v2.baselibrary.utils.basepika import BasePika
-from retry import retry
-
-logging_logger = logging.getLogger(__name__)
-
-
-class UseMq(object):
-
-    def __init__(self, queue, qos=1):
-        self.queue = queue
-        self.qos = qos
-        self.basepika = BasePika()
-        self.basepika.set_default()
-        self.basepika.connect()
-        self.basepika.create_channel()
-        self.basepika.queue_declare(queue=queue, durable=True)
-        self.basepika.basic_qos(qos)
-        self.properties = self.basepika.get_properties()
-
-    def re_conn(self):
-        """
-        重新连接
-        :return:
-        """
-        self.basepika.connect()
-        self.basepika.create_channel()
-        self.basepika.queue_declare(queue=self.queue, durable=True)
-        self.basepika.basic_qos(self.qos)
-
-    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
-    def get_mq(self):
-        try:
-            if self.basepika.channel.is_closed:
-                logging_logger.info("重连中......")
-                self.re_conn()
-                logging_logger.info("重连完成......")
-            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
-            self.basepika.start_get_msg()
-        except:
-            traceback.print_exc()
-            logging_logger.info("重连中......")
-            self.re_conn()
-
-    def callback(self, ch, method, properties, body):
-        # print(type(body))
-        # print(" [x] Received %r" % body)
-        # body = body.decode()
-        self.callback2(ch, method, properties, body)
-        if self.basepika.auto_ack is False:
-            self.basepika.basic_ack(ch, method)
-
-    def callback2(self, ch, method, properties, body):
-        pass
-
-    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
-    def send_mq(self, body, num=100):
-        try:
-            if self.basepika.get_queue_size(self.queue) < num:
-                self.basepika.easy_send_msg(routing_key=self.queue,
-                                            body=body,
-                                            properties=self.properties)
-                return True
-            else:
-                return False
-        except:
-            traceback.print_exc()
-            logging_logger.info("重连中......")
-            self.re_conn()
-            return False
-
-    def get_server_mq_num(self, num=100):
-        if self.basepika.get_queue_size(self.queue) < num:
-            return True
-        else:
-            return False
-
-    def easy_send_mq(self, body):
-        self.basepika.easy_send_msg(routing_key=self.queue,
-                                    body=body,
-                                    properties=self.properties)
+import logging
+import traceback
+
+from re_common.v2.baselibrary.utils.basepika import BasePika
+from retry import retry
+
+logging_logger = logging.getLogger(__name__)
+
+
+class UseMq(object):
+
+    def __init__(self, queue, qos=1):
+        self.queue = queue
+        self.qos = qos
+        self.basepika = BasePika()
+        self.basepika.set_default()
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=queue, durable=True)
+        self.basepika.basic_qos(qos)
+        self.properties = self.basepika.get_properties()
+
+    def re_conn(self):
+        """
+        重新连接
+        :return:
+        """
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=self.queue, durable=True)
+        self.basepika.basic_qos(self.qos)
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def get_mq(self):
+        try:
+            if self.basepika.channel.is_closed:
+                logging_logger.info("重连中......")
+                self.re_conn()
+                logging_logger.info("重连完成......")
+            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
+            self.basepika.start_get_msg()
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+
+    def callback(self, ch, method, properties, body):
+        # print(type(body))
+        # print(" [x] Received %r" % body)
+        # body = body.decode()
+        self.callback2(ch, method, properties, body)
+        if self.basepika.auto_ack is False:
+            self.basepika.basic_ack(ch, method)
+
+    def callback2(self, ch, method, properties, body):
+        pass
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def send_mq(self, body, num=100):
+        try:
+            if self.basepika.get_queue_size(self.queue) < num:
+                self.basepika.easy_send_msg(routing_key=self.queue,
+                                            body=body,
+                                            properties=self.properties)
+                return True
+            else:
+                return False
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+            return False
+
+    def get_server_mq_num(self, num=100):
+        if self.basepika.get_queue_size(self.queue) < num:
+            return True
+        else:
+            return False
+
+    def easy_send_mq(self, body):
+        self.basepika.easy_send_msg(routing_key=self.queue,
+                                    body=body,
+                                    properties=self.properties)