codosdk 1.0.53__tar.gz → 1.0.55__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codosdk-1.0.53 → codosdk-1.0.55}/PKG-INFO +1 -1
- {codosdk-1.0.53 → codosdk-1.0.55}/codosdk.egg-info/PKG-INFO +1 -1
- {codosdk-1.0.53 → codosdk-1.0.55}/setup.py +1 -1
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/agent_apis.py +9 -0
- codosdk-1.0.55/websdk2/cache_context.py +150 -0
- codosdk-1.0.55/websdk2/mqhelper.py +435 -0
- codosdk-1.0.53/websdk2/cache_context.py +0 -38
- codosdk-1.0.53/websdk2/mqhelper.py +0 -625
- {codosdk-1.0.53 → codosdk-1.0.55}/LICENSE +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/README.md +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/codosdk.egg-info/SOURCES.txt +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/codosdk.egg-info/dependency_links.txt +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/codosdk.egg-info/requires.txt +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/codosdk.egg-info/top_level.txt +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/opssdk/__init__.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/opssdk/utils/__init__.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/pyproject.toml +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/setup.cfg +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/__init__.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/api_set.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/__init__.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/admin_apis.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/cmdb_apis.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/kerrigan_apis.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/mgv4_apis.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/notice_apis.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/apis/task_apis.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/application.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/base_handler.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/cache.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/client.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/cloud/__init__.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/cloud/qcloud_api.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/cloud/ucloud_api.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/cloud_utils.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/configs.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/consts.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/crud_utils.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/db_context.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/error.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/fetch_coroutine.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/jwt_token.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/ldap.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/logger.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/model_utils.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/program.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/salt_api.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/sqlalchemy_pagination.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/tools.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/utils/__init__.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/utils/cc_crypto.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/utils/date_format.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/utils/pydantic_utils.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/utils/pydantic_utils_v2.py +0 -0
- {codosdk-1.0.53 → codosdk-1.0.55}/websdk2/web_logs.py +0 -0
|
@@ -9,7 +9,7 @@ desc : CODO SDK
|
|
|
9
9
|
import sys
|
|
10
10
|
from setuptools import setup, find_packages
|
|
11
11
|
|
|
12
|
-
VERSION = '1.0.
|
|
12
|
+
VERSION = '1.0.55'
|
|
13
13
|
|
|
14
14
|
if sys.version_info < (2, 7) or (3, 0) <= sys.version_info < (3, 6):
|
|
15
15
|
print('This program requires at least Python 2.7 or 3.6 to run.')
|
|
@@ -18,6 +18,7 @@ class AgentAPIS:
|
|
|
18
18
|
|
|
19
19
|
sync_files_register_url_v2 = f"{route_prefix}/v1/manager/agent/share_filev2/register"
|
|
20
20
|
sync_files_cancel_url_v2 = f"{route_prefix}/v1/manager/agent/share_filev2/cancel"
|
|
21
|
+
get_agent_task_result_url = f"{route_prefix}/v1/manager/agent/task/result'"
|
|
21
22
|
|
|
22
23
|
get_agent_list = dict(method='GET',
|
|
23
24
|
url=agent_list_url,
|
|
@@ -77,3 +78,11 @@ class AgentAPIS:
|
|
|
77
78
|
field_help={
|
|
78
79
|
},
|
|
79
80
|
description='云原生任务查询')
|
|
81
|
+
|
|
82
|
+
get_agent_task_result = dict(method='GET',
|
|
83
|
+
url=get_agent_task_result_url,
|
|
84
|
+
params={},
|
|
85
|
+
field_help={
|
|
86
|
+
'task_id': "提供任务ID"
|
|
87
|
+
},
|
|
88
|
+
description='云原生任务查询')
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Contact : 191715030@qq.com
|
|
5
|
+
Author : shenshuo
|
|
6
|
+
Date : 2018/11/26
|
|
7
|
+
Desc : Redis连接
|
|
8
|
+
"""
|
|
9
|
+
import threading
|
|
10
|
+
from typing import Dict, Optional
|
|
11
|
+
|
|
12
|
+
import redis
|
|
13
|
+
from .consts import const
|
|
14
|
+
from .configs import configs
|
|
15
|
+
|
|
16
|
+
_cache_conns: Dict[str, redis.Redis] = {}
|
|
17
|
+
_init_lock = threading.Lock()
|
|
18
|
+
|
|
19
|
+
# 固定默认值
|
|
20
|
+
_MAX_CONNECTIONS: int = 200
|
|
21
|
+
_POOL_TIMEOUT: float = 5.0
|
|
22
|
+
_SOCKET_CONNECT_TIMEOUT: float = 2.0
|
|
23
|
+
_SOCKET_TIMEOUT: float = 3.0
|
|
24
|
+
_RETRY_ON_TIMEOUT: bool = True
|
|
25
|
+
_HEALTH_CHECK_INTERVAL: int = 30
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _validate_basic_types(key: Optional[str], db: Optional[int]) -> None:
|
|
29
|
+
if key is not None and not isinstance(key, str):
|
|
30
|
+
raise TypeError(f"key 必须为 str 或 None,当前类型为 {type(key).__name__}")
|
|
31
|
+
if db is not None and not isinstance(db, int):
|
|
32
|
+
raise TypeError(f"db 必须为 int 或 None,当前类型为 {type(db).__name__}")
|
|
33
|
+
# 可选:校验 db 合法范围(通常 0-15,取决于服务端配置)
|
|
34
|
+
if isinstance(db, int) and db < 0:
|
|
35
|
+
raise ValueError(f"db 不能为负数,当前值为 {db}")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _extract_and_validate_cfg(key: str, db: Optional[int]) -> tuple[str, int, Optional[str], bool, bool, int]:
|
|
39
|
+
# 读取配置字典并进行存在性校验
|
|
40
|
+
redis_configs = configs[const.REDIS_CONFIG_ITEM]
|
|
41
|
+
if key not in redis_configs:
|
|
42
|
+
raise ValueError(f"Redis配置键 '{key}' 不存在")
|
|
43
|
+
cfg = redis_configs[key]
|
|
44
|
+
|
|
45
|
+
# 逐项提取并校验类型
|
|
46
|
+
host = cfg[const.RD_HOST_KEY]
|
|
47
|
+
port = cfg[const.RD_PORT_KEY]
|
|
48
|
+
password = cfg.get(const.RD_PASSWORD_KEY)
|
|
49
|
+
auth = cfg.get(const.RD_AUTH_KEY, bool(password))
|
|
50
|
+
decode_responses = cfg.get(const.RD_DECODE_RESPONSES, False)
|
|
51
|
+
db_num = db if db is not None else cfg[const.RD_DB_KEY]
|
|
52
|
+
|
|
53
|
+
if not isinstance(host, str) or not host:
|
|
54
|
+
raise TypeError(f"RD_HOST 必须为非空字符串,当前: {host!r}")
|
|
55
|
+
if not isinstance(port, int) or port <= 0:
|
|
56
|
+
raise TypeError(f"RD_PORT 必须为正整数,当前: {port!r}")
|
|
57
|
+
if password is not None and not isinstance(password, str):
|
|
58
|
+
raise TypeError(f"RD_PASSWORD 必须为 str 或 None,当前: {type(password).__name__}")
|
|
59
|
+
if not isinstance(auth, bool):
|
|
60
|
+
raise TypeError(f"RD_AUTH 必须为 bool,当前: {type(auth).__name__}")
|
|
61
|
+
if not isinstance(decode_responses, bool):
|
|
62
|
+
raise TypeError(f"RD_DECODE_RESPONSES 必须为 bool,当前: {type(decode_responses).__name__}")
|
|
63
|
+
if not isinstance(db_num, int) or db_num < 0:
|
|
64
|
+
raise TypeError(f"RD_DB 必须为非负整数,当前: {db_num!r}")
|
|
65
|
+
|
|
66
|
+
return host, port, password, auth, decode_responses, db_num
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _build_conn(
|
|
70
|
+
host: str,
|
|
71
|
+
port: int,
|
|
72
|
+
db_num: int,
|
|
73
|
+
password: Optional[str],
|
|
74
|
+
auth: bool,
|
|
75
|
+
decode_responses: bool,
|
|
76
|
+
cache_key: str,
|
|
77
|
+
) -> redis.Redis:
|
|
78
|
+
# 快速路径(无锁)
|
|
79
|
+
conn = _cache_conns.get(cache_key)
|
|
80
|
+
if conn is not None:
|
|
81
|
+
return conn
|
|
82
|
+
|
|
83
|
+
# 慢路径:加锁 + 双重检查
|
|
84
|
+
with _init_lock:
|
|
85
|
+
conn = _cache_conns.get(cache_key)
|
|
86
|
+
if conn is not None:
|
|
87
|
+
return conn
|
|
88
|
+
|
|
89
|
+
# 认证处理
|
|
90
|
+
if isinstance(password, str) and password == "":
|
|
91
|
+
password = None
|
|
92
|
+
if not auth and password:
|
|
93
|
+
auth = True
|
|
94
|
+
if auth and not password:
|
|
95
|
+
raise ValueError("Redis要求认证但未提供有效密码")
|
|
96
|
+
|
|
97
|
+
# 阻塞连接池 + 固定默认值
|
|
98
|
+
pool = redis.BlockingConnectionPool(
|
|
99
|
+
host=host,
|
|
100
|
+
port=port,
|
|
101
|
+
db=db_num,
|
|
102
|
+
password=password if auth else None,
|
|
103
|
+
decode_responses=decode_responses,
|
|
104
|
+
max_connections=_MAX_CONNECTIONS,
|
|
105
|
+
timeout=_POOL_TIMEOUT,
|
|
106
|
+
socket_connect_timeout=_SOCKET_CONNECT_TIMEOUT,
|
|
107
|
+
socket_timeout=_SOCKET_TIMEOUT,
|
|
108
|
+
retry_on_timeout=_RETRY_ON_TIMEOUT,
|
|
109
|
+
health_check_interval=_HEALTH_CHECK_INTERVAL,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
conn = redis.Redis(connection_pool=pool)
|
|
113
|
+
_cache_conns[cache_key] = conn
|
|
114
|
+
return conn
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cache_conn(key: Optional[str] = None, db: Optional[int] = None) -> redis.Redis:
|
|
118
|
+
"""
|
|
119
|
+
获取 Redis 客户端(遵循配置中的 RD_DECODE_RESPONSES)
|
|
120
|
+
- 老代码兼容:如果配置是 False,则返回 bytes;True 则返回 str
|
|
121
|
+
"""
|
|
122
|
+
_validate_basic_types(key, db)
|
|
123
|
+
|
|
124
|
+
# 默认 key
|
|
125
|
+
if not key:
|
|
126
|
+
from .consts import const as _const
|
|
127
|
+
key = _const.DEFAULT_RD_KEY
|
|
128
|
+
|
|
129
|
+
host, port, password, auth, decode_responses, db_num = _extract_and_validate_cfg(key, db)
|
|
130
|
+
cache_key = f"{key}:{db_num}"
|
|
131
|
+
return _build_conn(host, port, db_num, password, auth, decode_responses, cache_key)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def cache_conn_decode(key: Optional[str] = None, db: Optional[int] = None) -> redis.Redis:
|
|
135
|
+
"""
|
|
136
|
+
获取 Redis 客户端(强制 decode_responses=True)
|
|
137
|
+
- 所有返回值为 str
|
|
138
|
+
- 与 cache_conn 共存,使用独立的缓存键后缀,避免类型混用
|
|
139
|
+
"""
|
|
140
|
+
_validate_basic_types(key, db)
|
|
141
|
+
|
|
142
|
+
# 默认 key
|
|
143
|
+
if not key:
|
|
144
|
+
from .consts import const as _const
|
|
145
|
+
key = _const.DEFAULT_RD_KEY
|
|
146
|
+
|
|
147
|
+
host, port, password, auth, _decode_responses_cfg, db_num = _extract_and_validate_cfg(key, db)
|
|
148
|
+
cache_key = f"{key}:{db_num}:decode"
|
|
149
|
+
# 强制字符串返回
|
|
150
|
+
return _build_conn(host, port, db_num, password, auth, True, cache_key)
|
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
# !/usr/bin/env python
|
|
2
|
+
# -*-coding:utf-8-*-
|
|
3
|
+
"""
|
|
4
|
+
Author : ming
|
|
5
|
+
date : 2017/3/3 下午9:31
|
|
6
|
+
role : rabbitMQ 操作类
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import traceback
|
|
11
|
+
import pika
|
|
12
|
+
import time
|
|
13
|
+
import json
|
|
14
|
+
from typing import Optional, Tuple, Union, List
|
|
15
|
+
|
|
16
|
+
from pika.exceptions import AMQPConnectionError, ChannelClosedByBroker
|
|
17
|
+
from .consts import const
|
|
18
|
+
from .configs import configs
|
|
19
|
+
from .error import ConfigError
|
|
20
|
+
|
|
21
|
+
# 配置pika日志级别
|
|
22
|
+
logger = logging.getLogger('pika')
|
|
23
|
+
logger.setLevel(logging.ERROR)
|
|
24
|
+
|
|
25
|
+
# 屏蔽pika诊断日志噪音
|
|
26
|
+
pika_diagnostic_logger = logging.getLogger('pika.diagnostic_utils')
|
|
27
|
+
pika_diagnostic_logger.setLevel(logging.CRITICAL)
|
|
28
|
+
|
|
29
|
+
# 屏蔽pika适配器日志噪音
|
|
30
|
+
pika_adapter_logger = logging.getLogger('pika.adapters')
|
|
31
|
+
pika_adapter_logger.setLevel(logging.CRITICAL)
|
|
32
|
+
JsonOrBytes = Union[bytes, bytearray, str, dict, list]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MessageQueueBase:
|
|
36
|
+
def __init__(self, exchange, exchange_type, routing_key="", routing_keys=None,
|
|
37
|
+
queue_name="", no_ack=False, mq_key=const.DEFAULT_MQ_KEY):
|
|
38
|
+
|
|
39
|
+
mq_config = configs[const.MQ_CONFIG_ITEM].get(mq_key)
|
|
40
|
+
if not mq_config:
|
|
41
|
+
raise ConfigError(f"MQ config {mq_key} not found")
|
|
42
|
+
|
|
43
|
+
required_keys = [const.MQ_ADDR, const.MQ_PORT, const.MQ_VHOST, const.MQ_USER, const.MQ_PWD]
|
|
44
|
+
for k in required_keys:
|
|
45
|
+
if k not in mq_config:
|
|
46
|
+
raise ConfigError(k)
|
|
47
|
+
|
|
48
|
+
self.addr = mq_config[const.MQ_ADDR]
|
|
49
|
+
self.port = int(mq_config[const.MQ_PORT])
|
|
50
|
+
self.vhost = mq_config[const.MQ_VHOST]
|
|
51
|
+
self.user = mq_config[const.MQ_USER]
|
|
52
|
+
self.pwd = mq_config[const.MQ_PWD]
|
|
53
|
+
|
|
54
|
+
self.exchange = exchange
|
|
55
|
+
self.exchange_type = exchange_type
|
|
56
|
+
self.routing_key = routing_key
|
|
57
|
+
self.routing_keys = routing_keys or []
|
|
58
|
+
self.queue_name = queue_name
|
|
59
|
+
self.no_ack = no_ack
|
|
60
|
+
|
|
61
|
+
self.connection = None
|
|
62
|
+
self.channel = None
|
|
63
|
+
|
|
64
|
+
# ========== Context Manager ==========
|
|
65
|
+
def __enter__(self):
|
|
66
|
+
self.channel = self.create_channel()
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
70
|
+
self.close()
|
|
71
|
+
|
|
72
|
+
# ========== Connection / Channel ==========
|
|
73
|
+
def create_channel(self):
|
|
74
|
+
credentials = pika.PlainCredentials(self.user, self.pwd)
|
|
75
|
+
self.connection = pika.BlockingConnection(
|
|
76
|
+
pika.ConnectionParameters(self.addr, self.port, self.vhost, credentials=credentials)
|
|
77
|
+
)
|
|
78
|
+
self.channel = self.connection.channel()
|
|
79
|
+
return self.channel
|
|
80
|
+
|
|
81
|
+
def close(self):
|
|
82
|
+
if self.connection and not self.connection.is_closed:
|
|
83
|
+
self.connection.close()
|
|
84
|
+
|
|
85
|
+
def new_channel(self):
|
|
86
|
+
self.channel = self.create_channel()
|
|
87
|
+
return self
|
|
88
|
+
|
|
89
|
+
# ========== Consume ==========
|
|
90
|
+
def start_consuming(self, exchange_durable=False):
|
|
91
|
+
ch = self.create_channel()
|
|
92
|
+
self._declare_exchange(ch, exchange_durable)
|
|
93
|
+
queue_name = self._declare_and_bind_queue(ch)
|
|
94
|
+
|
|
95
|
+
ch.basic_qos(prefetch_count=1)
|
|
96
|
+
ch.basic_consume(queue=queue_name, on_message_callback=self.call_back, auto_ack=self.no_ack)
|
|
97
|
+
logger.info(f"[*] Queue {queue_name} started.")
|
|
98
|
+
ch.start_consuming()
|
|
99
|
+
|
|
100
|
+
def call_back(self, ch, method, properties, body):
|
|
101
|
+
try:
|
|
102
|
+
logger.info("Received message")
|
|
103
|
+
self.on_message(body)
|
|
104
|
+
if not self.no_ack:
|
|
105
|
+
ch.basic_ack(delivery_tag=method.delivery_tag)
|
|
106
|
+
except Exception:
|
|
107
|
+
logger.error(traceback.format_exc())
|
|
108
|
+
if not self.no_ack:
|
|
109
|
+
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
|
|
110
|
+
|
|
111
|
+
def on_message(self, body):
|
|
112
|
+
"""子类实现具体逻辑"""
|
|
113
|
+
raise NotImplementedError
|
|
114
|
+
|
|
115
|
+
# ========== Publish ==========
|
|
116
|
+
def publish_message(self, body, durable=True, exchange_durable=False):
|
|
117
|
+
if not self.channel:
|
|
118
|
+
self.create_channel()
|
|
119
|
+
|
|
120
|
+
self._declare_exchange(self.channel, exchange_durable)
|
|
121
|
+
queue_name = self._declare_and_bind_queue(self.channel)
|
|
122
|
+
|
|
123
|
+
props = pika.BasicProperties(delivery_mode=2) if durable else None
|
|
124
|
+
self.channel.basic_publish(
|
|
125
|
+
exchange=self.exchange,
|
|
126
|
+
routing_key=self.routing_key,
|
|
127
|
+
body=body,
|
|
128
|
+
properties=props,
|
|
129
|
+
)
|
|
130
|
+
logger.info(f"Publish message success: {body} -> queue {queue_name}")
|
|
131
|
+
|
|
132
|
+
# ========== Private Helpers ==========
|
|
133
|
+
def _declare_exchange(self, ch, exchange_durable):
|
|
134
|
+
ch.exchange_declare(exchange=self.exchange,
|
|
135
|
+
exchange_type=self.exchange_type,
|
|
136
|
+
durable=exchange_durable)
|
|
137
|
+
|
|
138
|
+
def _declare_and_bind_queue(self, ch):
|
|
139
|
+
if self.queue_name:
|
|
140
|
+
result = ch.queue_declare(queue=self.queue_name, durable=True)
|
|
141
|
+
else:
|
|
142
|
+
result = ch.queue_declare(queue="", exclusive=True, auto_delete=True)
|
|
143
|
+
|
|
144
|
+
queue_name = result.method.queue
|
|
145
|
+
if self.routing_keys:
|
|
146
|
+
for binding_key in self.routing_keys:
|
|
147
|
+
ch.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=binding_key)
|
|
148
|
+
else:
|
|
149
|
+
ch.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=self.routing_key)
|
|
150
|
+
return queue_name
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class MessageQueueBaseV2:
|
|
154
|
+
"""
|
|
155
|
+
消息队列基类:集成消费者和生产者功能。
|
|
156
|
+
线程模型:推荐每个线程创建独立实例,不要跨线程共享(BlockingConnection 非线程安全)。
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
def __init__(self, exchange: str, exchange_type: str = "direct", routing_key: str = "",
|
|
160
|
+
routing_keys: Optional[List[str]] = None, queue_name: str = "",
|
|
161
|
+
no_ack: bool = False, mq_key: str = const.DEFAULT_MQ_KEY):
|
|
162
|
+
|
|
163
|
+
# 读取配置
|
|
164
|
+
mq_config = configs[const.MQ_CONFIG_ITEM].get(mq_key)
|
|
165
|
+
if not mq_config:
|
|
166
|
+
raise ConfigError(f"MQ config {mq_key} not found")
|
|
167
|
+
|
|
168
|
+
required_keys = [const.MQ_ADDR, const.MQ_PORT, const.MQ_VHOST, const.MQ_USER, const.MQ_PWD]
|
|
169
|
+
for k in required_keys:
|
|
170
|
+
if k not in mq_config:
|
|
171
|
+
raise ConfigError(k)
|
|
172
|
+
|
|
173
|
+
self.addr = mq_config[const.MQ_ADDR]
|
|
174
|
+
self.port = int(mq_config[const.MQ_PORT])
|
|
175
|
+
self.vhost = mq_config[const.MQ_VHOST]
|
|
176
|
+
self.user = mq_config[const.MQ_USER]
|
|
177
|
+
self.pwd = mq_config[const.MQ_PWD]
|
|
178
|
+
|
|
179
|
+
# 拓扑与消费参数
|
|
180
|
+
self.exchange = exchange
|
|
181
|
+
self.exchange_type = exchange_type
|
|
182
|
+
self.routing_key = routing_key
|
|
183
|
+
self.routing_keys = routing_keys or []
|
|
184
|
+
self.queue_name = queue_name
|
|
185
|
+
self.no_ack = no_ack
|
|
186
|
+
|
|
187
|
+
# 连接/通道
|
|
188
|
+
self.connection: Optional[pika.BlockingConnection] = None
|
|
189
|
+
self.channel: Optional[pika.adapters.blocking_connection.BlockingChannel] = None
|
|
190
|
+
|
|
191
|
+
# 状态
|
|
192
|
+
self._returned = []
|
|
193
|
+
self._callbacks_ready = False
|
|
194
|
+
self._blocked = False
|
|
195
|
+
|
|
196
|
+
# 统一的连接参数(避免重复创建)
|
|
197
|
+
self._conn_params = pika.ConnectionParameters(
|
|
198
|
+
host=self.addr,
|
|
199
|
+
port=self.port,
|
|
200
|
+
virtual_host=self.vhost,
|
|
201
|
+
credentials=pika.PlainCredentials(self.user, self.pwd),
|
|
202
|
+
heartbeat=30,
|
|
203
|
+
blocked_connection_timeout=300,
|
|
204
|
+
connection_attempts=3,
|
|
205
|
+
retry_delay=2.0,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# ========== Context Manager ==========
|
|
209
|
+
def __enter__(self):
|
|
210
|
+
self._ensure_conn_chan(need_confirm=False)
|
|
211
|
+
return self
|
|
212
|
+
|
|
213
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
214
|
+
self.close()
|
|
215
|
+
|
|
216
|
+
# ========== 连接/通道管理 ==========
|
|
217
|
+
def _ensure_conn_chan(self, need_confirm: bool = False) -> None:
|
|
218
|
+
"""
|
|
219
|
+
确保连接和通道可用,并按需注册回调和确认机制。
|
|
220
|
+
|
|
221
|
+
参数:
|
|
222
|
+
- need_confirm: 是否需要发布确认(消费时 False,生产时 True)
|
|
223
|
+
"""
|
|
224
|
+
# 确保连接可用
|
|
225
|
+
if not (self.connection and self.connection.is_open):
|
|
226
|
+
self.connection = pika.BlockingConnection(self._conn_params)
|
|
227
|
+
self.connection.add_on_connection_blocked_callback(
|
|
228
|
+
lambda c, r: self._on_blocked(r)
|
|
229
|
+
)
|
|
230
|
+
self.connection.add_on_connection_unblocked_callback(
|
|
231
|
+
lambda c: self._on_unblocked()
|
|
232
|
+
)
|
|
233
|
+
self._callbacks_ready = False
|
|
234
|
+
|
|
235
|
+
# 确保通道可用
|
|
236
|
+
if not (self.channel and self.channel.is_open):
|
|
237
|
+
self.channel = self.connection.channel()
|
|
238
|
+
self._callbacks_ready = False
|
|
239
|
+
|
|
240
|
+
# 按需注册发布确认和 return 回调(仅注册一次)
|
|
241
|
+
if need_confirm and not self._callbacks_ready:
|
|
242
|
+
self.channel.confirm_delivery()
|
|
243
|
+
self._returned.clear()
|
|
244
|
+
self.channel.add_on_return_callback(
|
|
245
|
+
lambda ch, m, p, b: self._returned.append((m, p, b))
|
|
246
|
+
)
|
|
247
|
+
self._callbacks_ready = True
|
|
248
|
+
|
|
249
|
+
def _on_blocked(self, reason: str):
|
|
250
|
+
"""broker 阻塞回调(内存/磁盘告警)"""
|
|
251
|
+
self._blocked = True
|
|
252
|
+
logger.warning(f"Connection blocked: {reason}")
|
|
253
|
+
|
|
254
|
+
def _on_unblocked(self):
|
|
255
|
+
"""broker 解除阻塞回调"""
|
|
256
|
+
self._blocked = False
|
|
257
|
+
logger.info("Connection unblocked")
|
|
258
|
+
|
|
259
|
+
def _close_resources(self) -> None:
|
|
260
|
+
"""关闭连接和通道(先关通道,再关连接)"""
|
|
261
|
+
# 关闭通道
|
|
262
|
+
if self.channel:
|
|
263
|
+
try:
|
|
264
|
+
if self.channel.is_open:
|
|
265
|
+
self.channel.close()
|
|
266
|
+
except Exception:
|
|
267
|
+
pass
|
|
268
|
+
finally:
|
|
269
|
+
self.channel = None
|
|
270
|
+
|
|
271
|
+
# 关闭连接
|
|
272
|
+
if self.connection:
|
|
273
|
+
try:
|
|
274
|
+
if self.connection.is_open:
|
|
275
|
+
self.connection.close()
|
|
276
|
+
except Exception:
|
|
277
|
+
pass
|
|
278
|
+
finally:
|
|
279
|
+
self.connection = None
|
|
280
|
+
|
|
281
|
+
# 重置状态
|
|
282
|
+
self._callbacks_ready = False
|
|
283
|
+
self._returned.clear()
|
|
284
|
+
|
|
285
|
+
def close(self):
|
|
286
|
+
"""关闭连接和通道"""
|
|
287
|
+
self._close_resources()
|
|
288
|
+
|
|
289
|
+
def _reset_connection(self) -> None:
|
|
290
|
+
"""异常时重置连接(同 close)"""
|
|
291
|
+
self._close_resources()
|
|
292
|
+
|
|
293
|
+
# ========== 消费 ==========
|
|
294
|
+
def start_consuming(self, exchange_durable: bool = False):
|
|
295
|
+
"""开始消费消息"""
|
|
296
|
+
self._ensure_conn_chan(need_confirm=False)
|
|
297
|
+
self._declare_exchange(self.channel, exchange_durable)
|
|
298
|
+
queue_name = self._declare_and_bind_queue(self.channel)
|
|
299
|
+
|
|
300
|
+
self.channel.basic_qos(prefetch_count=1)
|
|
301
|
+
self.channel.basic_consume(queue=queue_name, on_message_callback=self._callback,
|
|
302
|
+
auto_ack=self.no_ack)
|
|
303
|
+
logger.info(f"[*] Queue {queue_name} started")
|
|
304
|
+
self.channel.start_consuming()
|
|
305
|
+
|
|
306
|
+
def _callback(self, ch, method, properties, body):
|
|
307
|
+
"""消息回调处理"""
|
|
308
|
+
try:
|
|
309
|
+
self.on_message(body)
|
|
310
|
+
if not self.no_ack:
|
|
311
|
+
ch.basic_ack(delivery_tag=method.delivery_tag)
|
|
312
|
+
except Exception:
|
|
313
|
+
logger.error("on_message error:\n%s", traceback.format_exc())
|
|
314
|
+
if not self.no_ack:
|
|
315
|
+
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
|
|
316
|
+
|
|
317
|
+
def on_message(self, body: bytes):
|
|
318
|
+
"""子类实现具体逻辑"""
|
|
319
|
+
raise NotImplementedError
|
|
320
|
+
|
|
321
|
+
# ========== 生产 ==========
|
|
322
|
+
@staticmethod
|
|
323
|
+
def _encode_body(body: JsonOrBytes, json_encode: bool) -> Tuple[bytes, str]:
|
|
324
|
+
"""
|
|
325
|
+
编码消息体并推断 content_type。
|
|
326
|
+
|
|
327
|
+
返回:(payload, content_type)
|
|
328
|
+
"""
|
|
329
|
+
if isinstance(body, (bytes, bytearray)):
|
|
330
|
+
return bytes(body), "application/octet-stream"
|
|
331
|
+
if isinstance(body, str):
|
|
332
|
+
return body.encode("utf-8"), "text/plain"
|
|
333
|
+
if json_encode:
|
|
334
|
+
return json.dumps(body, ensure_ascii=False).encode("utf-8"), "application/json"
|
|
335
|
+
raise TypeError(f"body type {type(body)} not supported")
|
|
336
|
+
|
|
337
|
+
def publish(self, exchange: str, routing_key: str, body: JsonOrBytes,
|
|
338
|
+
durable: bool = True, json_encode: bool = True, mandatory: bool = True,
|
|
339
|
+
max_retries: int = 6) -> None:
|
|
340
|
+
"""
|
|
341
|
+
发送消息(可靠性:发布确认 + mandatory + return 机制)。
|
|
342
|
+
|
|
343
|
+
参数:
|
|
344
|
+
- exchange: 交换机名
|
|
345
|
+
- routing_key: 路由键
|
|
346
|
+
- body: 消息体 (str/bytes/dict/list)
|
|
347
|
+
- durable: 是否持久化(默认 True)
|
|
348
|
+
- json_encode: 是否 JSON 编码(默认 True)
|
|
349
|
+
- mandatory: 是否强制要求有接收方(默认 True,无队列绑定时返回)
|
|
350
|
+
- max_retries: 最大重试次数(默认 6)
|
|
351
|
+
"""
|
|
352
|
+
# 检查 broker 是否阻塞
|
|
353
|
+
if self._blocked:
|
|
354
|
+
logger.warning("Broker is blocked, retrying may be needed")
|
|
355
|
+
|
|
356
|
+
# 编码消息
|
|
357
|
+
payload, actual_content_type = self._encode_body(body, json_encode)
|
|
358
|
+
props = pika.BasicProperties(
|
|
359
|
+
delivery_mode=2 if durable else 1, # 2=持久化, 1=临时
|
|
360
|
+
content_type=actual_content_type,
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
last_err = None
|
|
364
|
+
for attempt in range(1, max_retries + 1):
|
|
365
|
+
try:
|
|
366
|
+
# 确保连接和通道可用(需要确认机制)
|
|
367
|
+
self._ensure_conn_chan(need_confirm=True)
|
|
368
|
+
|
|
369
|
+
# 清空返回列表
|
|
370
|
+
self._returned.clear()
|
|
371
|
+
|
|
372
|
+
# 发送消息
|
|
373
|
+
self.channel.basic_publish(
|
|
374
|
+
exchange=exchange,
|
|
375
|
+
routing_key=routing_key,
|
|
376
|
+
body=payload,
|
|
377
|
+
properties=props,
|
|
378
|
+
mandatory=mandatory
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
# 检查消息是否被返回(无队列绑定)
|
|
382
|
+
if self._returned:
|
|
383
|
+
raise RuntimeError(f"Message unroutable: no binding for routing_key={routing_key}")
|
|
384
|
+
|
|
385
|
+
# logger.info(f"Message published: {exchange}/{routing_key}")
|
|
386
|
+
return
|
|
387
|
+
|
|
388
|
+
except (AMQPConnectionError, ChannelClosedByBroker) as e:
|
|
389
|
+
# 网络错误 → 重试
|
|
390
|
+
last_err = e
|
|
391
|
+
logger.warning(f"Connection error on attempt {attempt}/{max_retries}: {e}")
|
|
392
|
+
self._reset_connection()
|
|
393
|
+
if attempt < max_retries:
|
|
394
|
+
time.sleep(min(2 ** attempt, 5)) # 指数退避
|
|
395
|
+
|
|
396
|
+
except RuntimeError as e:
|
|
397
|
+
# 业务错误(不可路由等)→ 不重试
|
|
398
|
+
logger.error(f"Publish error: {e}")
|
|
399
|
+
raise
|
|
400
|
+
|
|
401
|
+
except Exception as e:
|
|
402
|
+
# 其他异常 → 重试
|
|
403
|
+
last_err = e
|
|
404
|
+
logger.error(f"Unexpected error on attempt {attempt}/{max_retries}: {e}")
|
|
405
|
+
self._reset_connection()
|
|
406
|
+
if attempt < max_retries:
|
|
407
|
+
time.sleep(min(2 ** attempt, 5))
|
|
408
|
+
|
|
409
|
+
raise last_err or RuntimeError("Unknown publish error")
|
|
410
|
+
|
|
411
|
+
# ========== 私有辅助方法 ==========
|
|
412
|
+
def _declare_exchange(self, ch, exchange_durable: bool):
|
|
413
|
+
"""声明交换机"""
|
|
414
|
+
ch.exchange_declare(exchange=self.exchange,
|
|
415
|
+
exchange_type=self.exchange_type,
|
|
416
|
+
durable=exchange_durable)
|
|
417
|
+
|
|
418
|
+
def _declare_and_bind_queue(self, ch):
|
|
419
|
+
"""声明队列并绑定"""
|
|
420
|
+
# 声明队列
|
|
421
|
+
if self.queue_name:
|
|
422
|
+
result = ch.queue_declare(queue=self.queue_name, durable=True)
|
|
423
|
+
else:
|
|
424
|
+
result = ch.queue_declare(queue="", exclusive=True, auto_delete=True)
|
|
425
|
+
|
|
426
|
+
queue_name = result.method.queue
|
|
427
|
+
|
|
428
|
+
# 绑定队列
|
|
429
|
+
if self.routing_keys:
|
|
430
|
+
for rk in self.routing_keys:
|
|
431
|
+
ch.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=rk)
|
|
432
|
+
else:
|
|
433
|
+
ch.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=self.routing_key)
|
|
434
|
+
|
|
435
|
+
return queue_name
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
|
-
"""
|
|
4
|
-
Contact : 191715030@qq.com
|
|
5
|
-
Author : shenshuo
|
|
6
|
-
Date : 2018/11/26
|
|
7
|
-
Desc :
|
|
8
|
-
"""
|
|
9
|
-
import redis
|
|
10
|
-
from .consts import const
|
|
11
|
-
from .configs import configs
|
|
12
|
-
|
|
13
|
-
cache_conns = {}
|
|
14
|
-
|
|
15
|
-
def cache_conn(key=None, db=None):
|
|
16
|
-
redis_configs = configs[const.REDIS_CONFIG_ITEM]
|
|
17
|
-
if not key:
|
|
18
|
-
key = const.DEFAULT_RD_KEY
|
|
19
|
-
for config_key, redis_config in redis_configs.items():
|
|
20
|
-
auth = redis_config[const.RD_AUTH_KEY]
|
|
21
|
-
host = redis_config[const.RD_HOST_KEY]
|
|
22
|
-
port = redis_config[const.RD_PORT_KEY]
|
|
23
|
-
password = redis_config[const.RD_PASSWORD_KEY]
|
|
24
|
-
if db:
|
|
25
|
-
db = db
|
|
26
|
-
else:
|
|
27
|
-
db = redis_config[const.RD_DB_KEY]
|
|
28
|
-
return_utf8 = False
|
|
29
|
-
if const.RD_DECODE_RESPONSES in redis_config:
|
|
30
|
-
return_utf8 = redis_config[const.RD_DECODE_RESPONSES]
|
|
31
|
-
|
|
32
|
-
if auth:
|
|
33
|
-
redis_pool = redis.ConnectionPool(host=host, port=port, db=db, password=password,
|
|
34
|
-
decode_responses=return_utf8)
|
|
35
|
-
else:
|
|
36
|
-
redis_pool = redis.ConnectionPool(host=host, port=port, db=db, decode_responses=return_utf8)
|
|
37
|
-
cache_conns[config_key] = redis.StrictRedis(connection_pool=redis_pool)
|
|
38
|
-
return cache_conns[key]
|