gomyck-tools 1.5.3__py3-none-any.whl → 1.5.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctools/authcode_validator.py +429 -0
- ctools/cipher/sm_util.py +0 -1
- ctools/database/database.py +27 -9
- ctools/ml/image_process.py +121 -0
- ctools/ml/img_extractor.py +59 -0
- ctools/ml/ppi.py +276 -0
- ctools/stream/ckafka.py +2 -2
- ctools/sys_log.py +151 -77
- ctools/util/jb_cut.py +0 -1
- ctools/web/bottle_web_base.py +1 -0
- ctools/web/bottle_webserver.py +11 -7
- ctools/web/bottle_websocket.py +1 -1
- ctools/web/ctoken.py +1 -1
- {gomyck_tools-1.5.3.dist-info → gomyck_tools-1.5.5.dist-info}/METADATA +15 -12
- {gomyck_tools-1.5.3.dist-info → gomyck_tools-1.5.5.dist-info}/RECORD +18 -14
- {gomyck_tools-1.5.3.dist-info → gomyck_tools-1.5.5.dist-info}/WHEEL +1 -1
- {gomyck_tools-1.5.3.dist-info → gomyck_tools-1.5.5.dist-info}/licenses/LICENSE +0 -0
- {gomyck_tools-1.5.3.dist-info → gomyck_tools-1.5.5.dist-info}/top_level.txt +0 -0
ctools/ml/ppi.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
from io import BytesIO
|
|
2
|
+
from queue import Queue
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import yaml
|
|
6
|
+
from PIL import Image
|
|
7
|
+
from paddle.inference import Config, create_predictor
|
|
8
|
+
|
|
9
|
+
from ctools import path_info
|
|
10
|
+
from ctools.ml.image_process import preprocess
|
|
11
|
+
from ctools.ml.img_extractor import ClassRegionBase64ExtractorPIL
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class PaddlePredictorPool:
|
|
15
|
+
"""Predictor 池,用于多线程安全推理"""
|
|
16
|
+
def __init__(self, config_path, pool_size: int = 4):
|
|
17
|
+
"""
|
|
18
|
+
初始化预测器池
|
|
19
|
+
Note: 每个 Config 对象只能创建一个 predictor,所以我们需要保存 config_path
|
|
20
|
+
"""
|
|
21
|
+
self.config_path = config_path
|
|
22
|
+
self.pool = Queue()
|
|
23
|
+
self._init_pool(pool_size)
|
|
24
|
+
|
|
25
|
+
def _load_config_yaml(self):
|
|
26
|
+
"""加载 yaml 配置"""
|
|
27
|
+
with open(self.config_path, "r", encoding="utf-8") as f:
|
|
28
|
+
return yaml.safe_load(f)
|
|
29
|
+
|
|
30
|
+
def _create_config(self):
|
|
31
|
+
"""为新的 predictor 创建一个新的 Config 对象"""
|
|
32
|
+
cfg = self._load_config_yaml()
|
|
33
|
+
model_dir = cfg.get("MODEL_DIR", "")
|
|
34
|
+
model_file = cfg.get("MODEL_FILE", "")
|
|
35
|
+
if not model_file:
|
|
36
|
+
model_dir = path_info.get_app_path('mod/model.pdmodel')
|
|
37
|
+
params_file = cfg.get("PARAMS_FILE", "")
|
|
38
|
+
if not params_file:
|
|
39
|
+
model_dir = path_info.get_app_path('mod/model.pdiparams')
|
|
40
|
+
use_gpu = cfg.get("USE_GPU", False)
|
|
41
|
+
|
|
42
|
+
if model_dir:
|
|
43
|
+
config = Config(model_dir)
|
|
44
|
+
else:
|
|
45
|
+
config = Config(model_file, params_file)
|
|
46
|
+
|
|
47
|
+
config.enable_memory_optim()
|
|
48
|
+
|
|
49
|
+
if use_gpu:
|
|
50
|
+
config.enable_use_gpu(1000, 0)
|
|
51
|
+
else:
|
|
52
|
+
config.set_cpu_math_library_num_threads(4)
|
|
53
|
+
config.enable_mkldnn()
|
|
54
|
+
|
|
55
|
+
return config
|
|
56
|
+
|
|
57
|
+
def _init_pool(self, pool_size: int):
|
|
58
|
+
"""初始化池中的所有 predictor"""
|
|
59
|
+
for _ in range(pool_size):
|
|
60
|
+
config = self._create_config()
|
|
61
|
+
predictor = create_predictor(config)
|
|
62
|
+
self.pool.put(predictor)
|
|
63
|
+
|
|
64
|
+
def acquire(self, timeout=None):
|
|
65
|
+
"""从池中获取一个 predictor"""
|
|
66
|
+
return self.pool.get(timeout=timeout)
|
|
67
|
+
|
|
68
|
+
def release(self, predictor):
|
|
69
|
+
"""将 predictor 放回池中"""
|
|
70
|
+
self.pool.put(predictor)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class PaddleInferenceEngine:
|
|
74
|
+
def __init__(self, config_path, pool_size=4):
|
|
75
|
+
self.config_path = config_path
|
|
76
|
+
self.cfg = self._load_config(config_path)
|
|
77
|
+
self.predictor_pool = PaddlePredictorPool(config_path, pool_size=pool_size)
|
|
78
|
+
|
|
79
|
+
def _load_config(self, config_path):
|
|
80
|
+
with open(config_path, "r", encoding="utf-8") as f:
|
|
81
|
+
return yaml.safe_load(f)
|
|
82
|
+
|
|
83
|
+
def predict(self, inputs, timeout=None):
|
|
84
|
+
"""线程安全预测"""
|
|
85
|
+
predictor = self.predictor_pool.acquire(timeout=timeout)
|
|
86
|
+
try:
|
|
87
|
+
input_names = predictor.get_input_names()
|
|
88
|
+
for name in input_names:
|
|
89
|
+
if name not in inputs:
|
|
90
|
+
raise ValueError(f"缺少模型输入: {name}")
|
|
91
|
+
tensor = predictor.get_input_handle(name)
|
|
92
|
+
data = inputs[name]
|
|
93
|
+
tensor.reshape(data.shape)
|
|
94
|
+
tensor.copy_from_cpu(data)
|
|
95
|
+
predictor.run()
|
|
96
|
+
outputs = []
|
|
97
|
+
for name in predictor.get_output_names():
|
|
98
|
+
out = predictor.get_output_handle(name)
|
|
99
|
+
outputs.append(out.copy_to_cpu())
|
|
100
|
+
return outputs
|
|
101
|
+
finally:
|
|
102
|
+
self.predictor_pool.release(predictor)
|
|
103
|
+
|
|
104
|
+
def predict_image(self, img, im_size=320):
|
|
105
|
+
if isinstance(img, bytes):
|
|
106
|
+
img = Image.open(BytesIO(img)).convert("RGB")
|
|
107
|
+
elif isinstance(img, Image.Image):
|
|
108
|
+
img = img.convert("RGB")
|
|
109
|
+
elif isinstance(img, np.ndarray):
|
|
110
|
+
pass
|
|
111
|
+
else:
|
|
112
|
+
raise ValueError("Unsupported image type for predict_image")
|
|
113
|
+
orig_img_np = np.array(img) if not isinstance(img, np.ndarray) else img
|
|
114
|
+
data = preprocess(orig_img_np, im_size)
|
|
115
|
+
scale_factor = np.array([im_size * 1. / orig_img_np.shape[0], im_size * 1. / orig_img_np.shape[1]]).reshape((1, 2)).astype(np.float32)
|
|
116
|
+
im_shape = np.array([im_size, im_size]).reshape((1, 2)).astype(np.float32)
|
|
117
|
+
outputs = self.predict({"image": data, "im_shape": im_shape, "scale_factor": scale_factor})
|
|
118
|
+
return outputs, scale_factor, im_size
|
|
119
|
+
|
|
120
|
+
def predict_image_and_extract(self, img, im_size=320, class_names=None, target_classes=None, threshold=0.3):
|
|
121
|
+
"""预测并提取检测区域"""
|
|
122
|
+
raw_outputs, scale_factor, im_size_ret = self.predict_image(img, im_size=im_size)
|
|
123
|
+
|
|
124
|
+
# 转换为 PIL Image
|
|
125
|
+
if isinstance(img, bytes):
|
|
126
|
+
pil_img = Image.open(BytesIO(img)).convert("RGB")
|
|
127
|
+
elif isinstance(img, Image.Image):
|
|
128
|
+
pil_img = img.convert("RGB")
|
|
129
|
+
elif isinstance(img, np.ndarray):
|
|
130
|
+
pil_img = Image.fromarray(img.astype('uint8')).convert("RGB")
|
|
131
|
+
else:
|
|
132
|
+
raise ValueError("Unsupported image type")
|
|
133
|
+
|
|
134
|
+
# 提取检测区域
|
|
135
|
+
extractor = ClassRegionBase64ExtractorPIL(class_names or [], target_classes=target_classes, threshold=threshold)
|
|
136
|
+
detection_results = raw_outputs[0]
|
|
137
|
+
return extractor.extract(pil_img, detection_results, scale_factor=scale_factor, im_size=im_size_ret)
|
|
138
|
+
|
|
139
|
+
@staticmethod
|
|
140
|
+
def _nms_detections(detections, iou_threshold=0.5):
|
|
141
|
+
if len(detections) == 0:
|
|
142
|
+
return detections
|
|
143
|
+
dets = np.array(detections, dtype=np.float32)
|
|
144
|
+
scores = dets[:, 1]
|
|
145
|
+
sorted_idx = np.argsort(-scores)
|
|
146
|
+
keep = []
|
|
147
|
+
while len(sorted_idx) > 0:
|
|
148
|
+
current_idx = sorted_idx[0]
|
|
149
|
+
keep.append(current_idx)
|
|
150
|
+
if len(sorted_idx) == 1:
|
|
151
|
+
break
|
|
152
|
+
current_box = dets[current_idx, 2:6]
|
|
153
|
+
other_boxes = dets[sorted_idx[1:], 2:6]
|
|
154
|
+
x1_inter = np.maximum(current_box[0], other_boxes[:, 0])
|
|
155
|
+
y1_inter = np.maximum(current_box[1], other_boxes[:, 1])
|
|
156
|
+
x2_inter = np.minimum(current_box[2], other_boxes[:, 2])
|
|
157
|
+
y2_inter = np.minimum(current_box[3], other_boxes[:, 3])
|
|
158
|
+
inter_area = np.maximum(0, x2_inter - x1_inter) * np.maximum(0, y2_inter - y1_inter)
|
|
159
|
+
area_current = (current_box[2] - current_box[0]) * (current_box[3] - current_box[1])
|
|
160
|
+
area_others = (other_boxes[:, 2] - other_boxes[:, 0]) * (other_boxes[:, 3] - other_boxes[:, 1])
|
|
161
|
+
union_area = area_current + area_others - inter_area
|
|
162
|
+
iou = inter_area / (union_area + 1e-6)
|
|
163
|
+
valid_idx = np.where(iou < iou_threshold)[0] + 1
|
|
164
|
+
sorted_idx = sorted_idx[valid_idx]
|
|
165
|
+
return dets[keep].tolist()
|
|
166
|
+
|
|
167
|
+
def predict_image_tiled(self, img, im_size=320, tile_overlap=0.2, class_names=None, target_classes=None, threshold=0.3, nms_iou=0.5):
|
|
168
|
+
"""
|
|
169
|
+
Tiled prediction for large images (2K, 4K, etc).
|
|
170
|
+
Splits image into overlapping tiles, predicts each tile, merges results.
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
img: PIL Image, numpy array, or bytes
|
|
174
|
+
im_size: model training resolution (e.g., 320)
|
|
175
|
+
tile_overlap: overlap ratio between tiles (0.0-1.0), default 0.2
|
|
176
|
+
class_names: list of class names
|
|
177
|
+
target_classes: list of target classes to extract
|
|
178
|
+
threshold: confidence threshold
|
|
179
|
+
nms_iou: IoU threshold for NMS merging
|
|
180
|
+
|
|
181
|
+
Returns:
|
|
182
|
+
extracted_outputs: list of dicts with extracted regions (with coordinates mapped to original image)
|
|
183
|
+
all_detections: list of raw detections [cat_id, score, xmin, ymin, xmax, ymax] (original image coords)
|
|
184
|
+
"""
|
|
185
|
+
# Convert input to numpy array
|
|
186
|
+
if isinstance(img, bytes):
|
|
187
|
+
pil_img = Image.open(BytesIO(img)).convert("RGB")
|
|
188
|
+
img_np = np.array(pil_img)
|
|
189
|
+
elif isinstance(img, Image.Image):
|
|
190
|
+
img_np = np.array(img.convert("RGB"))
|
|
191
|
+
elif isinstance(img, np.ndarray):
|
|
192
|
+
img_np = img
|
|
193
|
+
else:
|
|
194
|
+
raise ValueError("Unsupported image type for predict_image_tiled")
|
|
195
|
+
orig_h, orig_w = img_np.shape[:2]
|
|
196
|
+
# Calculate tile parameters
|
|
197
|
+
stride = int(im_size * (1 - tile_overlap))
|
|
198
|
+
stride = max(1, stride)
|
|
199
|
+
# Generate tile coordinates
|
|
200
|
+
tiles = []
|
|
201
|
+
y_start = 0
|
|
202
|
+
while y_start < orig_h:
|
|
203
|
+
y_end = min(y_start + im_size, orig_h)
|
|
204
|
+
# If last tile doesn't cover the bottom, adjust
|
|
205
|
+
if y_end == orig_h and y_start > 0:
|
|
206
|
+
y_start = max(0, orig_h - im_size)
|
|
207
|
+
y_end = orig_h
|
|
208
|
+
x_start = 0
|
|
209
|
+
while x_start < orig_w:
|
|
210
|
+
x_end = min(x_start + im_size, orig_w)
|
|
211
|
+
# If last tile doesn't cover the right, adjust
|
|
212
|
+
if x_end == orig_w and x_start > 0:
|
|
213
|
+
x_start = max(0, orig_w - im_size)
|
|
214
|
+
x_end = orig_w
|
|
215
|
+
tiles.append((x_start, y_start, x_end, y_end))
|
|
216
|
+
x_start += stride
|
|
217
|
+
if x_end == orig_w:
|
|
218
|
+
break
|
|
219
|
+
y_start += stride
|
|
220
|
+
if y_end == orig_h:
|
|
221
|
+
break
|
|
222
|
+
# Predict each tile and collect detections
|
|
223
|
+
all_detections = []
|
|
224
|
+
for x_start, y_start, x_end, y_end in tiles:
|
|
225
|
+
tile_img = img_np[y_start:y_end, x_start:x_end]
|
|
226
|
+
# Pad tile if smaller than im_size
|
|
227
|
+
if tile_img.shape[0] < im_size or tile_img.shape[1] < im_size:
|
|
228
|
+
pad_h = im_size - tile_img.shape[0]
|
|
229
|
+
pad_w = im_size - tile_img.shape[1]
|
|
230
|
+
tile_img = np.pad(tile_img, ((0, pad_h), (0, pad_w), (0, 0)), mode='constant', constant_values=0)
|
|
231
|
+
# Run inference on tile
|
|
232
|
+
try:
|
|
233
|
+
tile_outputs, _, _ = self.predict_image(tile_img, im_size=im_size)
|
|
234
|
+
tile_detections = tile_outputs[0]
|
|
235
|
+
# Map coordinates back to original image
|
|
236
|
+
for det in tile_detections:
|
|
237
|
+
cat_id, score = det[0], det[1]
|
|
238
|
+
xmin, ymin, xmax, ymax = det[2], det[3], det[4], det[5]
|
|
239
|
+
# Scale from model resolution to tile resolution
|
|
240
|
+
tile_h, tile_w = y_end - y_start, x_end - x_start
|
|
241
|
+
xmin_tile = xmin * tile_w / im_size
|
|
242
|
+
ymin_tile = ymin * tile_h / im_size
|
|
243
|
+
xmax_tile = xmax * tile_w / im_size
|
|
244
|
+
ymax_tile = ymax * tile_h / im_size
|
|
245
|
+
# Translate to original image coordinates
|
|
246
|
+
xmin_orig = xmin_tile + x_start
|
|
247
|
+
ymin_orig = ymin_tile + y_start
|
|
248
|
+
xmax_orig = xmax_tile + x_start
|
|
249
|
+
ymax_orig = ymax_tile + y_start
|
|
250
|
+
# Clip to image bounds
|
|
251
|
+
xmin_orig = max(0, min(xmin_orig, orig_w))
|
|
252
|
+
ymin_orig = max(0, min(ymin_orig, orig_h))
|
|
253
|
+
xmax_orig = max(0, min(xmax_orig, orig_w))
|
|
254
|
+
ymax_orig = max(0, min(ymax_orig, orig_h))
|
|
255
|
+
all_detections.append([cat_id, score, xmin_orig, ymin_orig, xmax_orig, ymax_orig])
|
|
256
|
+
except Exception as e:
|
|
257
|
+
print(f"Error processing tile {(x_start, y_start, x_end, y_end)}: {e}")
|
|
258
|
+
continue
|
|
259
|
+
# Apply NMS to merge duplicate detections
|
|
260
|
+
merged_detections = self._nms_detections(all_detections, iou_threshold=nms_iou)
|
|
261
|
+
|
|
262
|
+
# Extract regions using the merged detections
|
|
263
|
+
if isinstance(img, bytes):
|
|
264
|
+
pil_img = Image.open(BytesIO(img)).convert("RGB")
|
|
265
|
+
elif isinstance(img, Image.Image):
|
|
266
|
+
pil_img = img.convert("RGB")
|
|
267
|
+
elif isinstance(img, np.ndarray):
|
|
268
|
+
pil_img = Image.fromarray(img_np.astype('uint8')).convert("RGB")
|
|
269
|
+
else:
|
|
270
|
+
raise ValueError("Unsupported image type")
|
|
271
|
+
|
|
272
|
+
# Create a dummy scale_factor (1:1 since we're already in original coordinates)
|
|
273
|
+
scale_factor = np.array([[1.0, 1.0]], dtype=np.float32)
|
|
274
|
+
extractor = ClassRegionBase64ExtractorPIL(class_names or [], target_classes=target_classes, threshold=threshold)
|
|
275
|
+
extracted = extractor.extract(pil_img, merged_detections, scale_factor=scale_factor, im_size=orig_h)
|
|
276
|
+
return extracted, merged_detections
|
ctools/stream/ckafka.py
CHANGED
|
@@ -15,7 +15,7 @@ from ctools.cjson import dumps
|
|
|
15
15
|
import time
|
|
16
16
|
from datetime import datetime
|
|
17
17
|
|
|
18
|
-
from ctools import thread_pool,
|
|
18
|
+
from ctools import thread_pool, cid
|
|
19
19
|
from ctools.ckafka import CKafka
|
|
20
20
|
|
|
21
21
|
c = CKafka(kafka_url='192.168.3.160:9094', secure=True)
|
|
@@ -27,7 +27,7 @@ def send_msg():
|
|
|
27
27
|
while True:
|
|
28
28
|
command = input('发送消息: Y/n \n')
|
|
29
29
|
if command.strip() not in ['N', 'n']:
|
|
30
|
-
producer.send_msg('jqxx', '{{"jqid": "{}", "xxxx": "{}"}}'.format(
|
|
30
|
+
producer.send_msg('jqxx', '{{"jqid": "{}", "xxxx": "{}"}}'.format(cid.get_snowflake_id(), datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')))
|
|
31
31
|
else:
|
|
32
32
|
break
|
|
33
33
|
|
ctools/sys_log.py
CHANGED
|
@@ -1,96 +1,170 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import os
|
|
3
1
|
import sys
|
|
2
|
+
import os
|
|
4
3
|
import time
|
|
5
|
-
|
|
6
|
-
from ctools import call, path_info
|
|
7
|
-
|
|
8
|
-
clog: logging.Logger = None
|
|
9
|
-
flog: logging.Logger = None
|
|
10
|
-
|
|
11
|
-
neglect_keywords = [
|
|
12
|
-
"OPTIONS",
|
|
13
|
-
]
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
# 文件日志
|
|
17
|
-
@call.once
|
|
18
|
-
def _file_log(sys_log_path: str = './', log_level: int = logging.INFO, mixin: bool = False) -> logging:
|
|
19
|
-
try:
|
|
20
|
-
os.mkdir(sys_log_path)
|
|
21
|
-
except Exception:
|
|
22
|
-
pass
|
|
23
|
-
log_file = sys_log_path + os.path.sep + "log-" + time.strftime("%Y-%m-%d-%H", time.localtime(time.time())) + ".log"
|
|
24
|
-
if mixin:
|
|
25
|
-
handlers = [logging.FileHandler(filename=log_file, encoding='utf-8'), logging.StreamHandler()]
|
|
26
|
-
else:
|
|
27
|
-
handlers = [logging.FileHandler(filename=log_file, encoding='utf-8')]
|
|
28
|
-
logging.basicConfig(level=log_level,
|
|
29
|
-
format='%(asctime)s | %(levelname)-5s | T%(thread)d | %(module)s.%(funcName)s:%(lineno)d: %(message)s',
|
|
30
|
-
datefmt='%Y%m%d%H%M%S',
|
|
31
|
-
handlers=handlers)
|
|
32
|
-
logger = logging.getLogger('ck-flog')
|
|
33
|
-
return logger
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# 控制台日志
|
|
37
|
-
@call.once
|
|
38
|
-
def _console_log(log_level: int = logging.INFO) -> logging:
|
|
39
|
-
handler = logging.StreamHandler()
|
|
40
|
-
logging.basicConfig(level=log_level,
|
|
41
|
-
format='%(asctime)s | %(levelname)-5s | T%(thread)d | %(name)s | %(module)s.%(funcName)s:%(lineno)d: %(message)s',
|
|
42
|
-
datefmt='%Y%m%d%H%M%S',
|
|
43
|
-
handlers=[handler])
|
|
44
|
-
logger = logging.getLogger('ck-clog')
|
|
45
|
-
return logger
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
import io
|
|
49
4
|
import logging
|
|
5
|
+
from typing import Protocol
|
|
50
6
|
|
|
7
|
+
from ctools import call
|
|
51
8
|
|
|
52
|
-
class StreamToLogger(io.StringIO):
|
|
53
|
-
def __init__(self, logger: logging.Logger, level: int = logging.INFO):
|
|
54
|
-
super().__init__()
|
|
55
|
-
self.logger = logger
|
|
56
|
-
self.level = level
|
|
57
|
-
self._buffer = ''
|
|
58
9
|
|
|
59
|
-
|
|
60
|
-
|
|
10
|
+
# =========================
|
|
11
|
+
# 日志接口(IDE 提示)
|
|
12
|
+
# =========================
|
|
13
|
+
class LoggerProtocol(Protocol):
|
|
14
|
+
def debug(self, msg: str, *args, **kwargs) -> None: ...
|
|
15
|
+
def info(self, msg: str, *args, **kwargs) -> None: ...
|
|
16
|
+
def warning(self, msg: str, *args, **kwargs) -> None: ...
|
|
17
|
+
def error(self, msg: str, *args, **kwargs) -> None: ...
|
|
18
|
+
def critical(self, msg: str, *args, **kwargs) -> None: ...
|
|
19
|
+
def exception(self, msg: str, *args, **kwargs) -> None: ...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _LazyLogger:
|
|
23
|
+
def __getattr__(self, item):
|
|
24
|
+
raise RuntimeError("Logger not initialized, call init_log() first")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
flog: LoggerProtocol = _LazyLogger()
|
|
28
|
+
clog: LoggerProtocol = _LazyLogger()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# =========================
|
|
32
|
+
# TeeStream
|
|
33
|
+
# =========================
|
|
34
|
+
class TeeStream:
|
|
35
|
+
def __init__(self, *streams):
|
|
36
|
+
self.streams = streams
|
|
37
|
+
|
|
38
|
+
def write(self, data):
|
|
39
|
+
if not data:
|
|
61
40
|
return
|
|
62
|
-
self.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
try:
|
|
68
|
-
self.logger.log(self.level, line.strip(), stacklevel=3)
|
|
69
|
-
except Exception:
|
|
70
|
-
self.logger.log(self.level, line.strip())
|
|
71
|
-
self._buffer = ''
|
|
41
|
+
for s in self.streams:
|
|
42
|
+
try:
|
|
43
|
+
s.write(data)
|
|
44
|
+
except Exception:
|
|
45
|
+
pass
|
|
72
46
|
|
|
73
47
|
def flush(self):
|
|
74
|
-
|
|
48
|
+
for s in self.streams:
|
|
75
49
|
try:
|
|
76
|
-
|
|
50
|
+
s.flush()
|
|
77
51
|
except Exception:
|
|
78
|
-
|
|
79
|
-
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
def isatty(self):
|
|
55
|
+
return any(getattr(s, "isatty", lambda: False)() for s in self.streams)
|
|
80
56
|
|
|
81
57
|
def fileno(self):
|
|
82
|
-
|
|
58
|
+
for s in self.streams:
|
|
59
|
+
try:
|
|
60
|
+
return s.fileno()
|
|
61
|
+
except Exception:
|
|
62
|
+
pass
|
|
63
|
+
raise OSError
|
|
83
64
|
|
|
84
65
|
|
|
66
|
+
# =========================
|
|
67
|
+
# print → 文件
|
|
68
|
+
# =========================
|
|
69
|
+
class PrintToFile:
|
|
70
|
+
def __init__(self, file_handler: logging.Handler, level=logging.INFO):
|
|
71
|
+
self.handler = file_handler
|
|
72
|
+
self.level = level
|
|
73
|
+
self._buffer = ""
|
|
74
|
+
|
|
75
|
+
def write(self, msg):
|
|
76
|
+
if not msg:
|
|
77
|
+
return
|
|
78
|
+
self._buffer += msg
|
|
79
|
+
while "\n" in self._buffer:
|
|
80
|
+
line, self._buffer = self._buffer.split("\n", 1)
|
|
81
|
+
line = line.rstrip()
|
|
82
|
+
if line:
|
|
83
|
+
record = logging.LogRecord(
|
|
84
|
+
name="print",
|
|
85
|
+
level=self.level,
|
|
86
|
+
pathname="",
|
|
87
|
+
lineno=0,
|
|
88
|
+
msg=line,
|
|
89
|
+
args=(),
|
|
90
|
+
exc_info=None,
|
|
91
|
+
)
|
|
92
|
+
self.handler.emit(record)
|
|
93
|
+
|
|
94
|
+
def flush(self):
|
|
95
|
+
if self._buffer.strip():
|
|
96
|
+
record = logging.LogRecord(
|
|
97
|
+
name="print",
|
|
98
|
+
level=self.level,
|
|
99
|
+
pathname="",
|
|
100
|
+
lineno=0,
|
|
101
|
+
msg=self._buffer.strip(),
|
|
102
|
+
args=(),
|
|
103
|
+
exc_info=None,
|
|
104
|
+
)
|
|
105
|
+
self.handler.emit(record)
|
|
106
|
+
self._buffer = ""
|
|
107
|
+
|
|
108
|
+
def isatty(self):
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# =========================
|
|
113
|
+
# 初始化日志
|
|
114
|
+
# =========================
|
|
85
115
|
@call.init
|
|
86
|
-
def
|
|
116
|
+
def init_log():
|
|
87
117
|
global flog, clog
|
|
88
|
-
flog = _file_log(path_info.get_user_work_path(".ck/ck-py-log", mkdir=True), mixin=True, log_level=logging.DEBUG)
|
|
89
|
-
clog = _console_log()
|
|
90
|
-
sys.stdout = StreamToLogger(flog, level=logging.INFO)
|
|
91
|
-
sys.stderr = StreamToLogger(flog, level=logging.ERROR)
|
|
92
118
|
|
|
119
|
+
# 绝对路径
|
|
120
|
+
home_dir = os.path.expanduser("~")
|
|
121
|
+
log_dir = os.path.join(home_dir, ".ck", "ck-py-log")
|
|
122
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
123
|
+
|
|
124
|
+
log_file = os.path.join(
|
|
125
|
+
log_dir, f"log-{time.strftime('%Y-%m-%d-%H')}.log"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
formatter = logging.Formatter(
|
|
129
|
+
"%(asctime)s | %(levelname)-8s | %(name)s:%(lineno)d - %(message)s"
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# ===== 文件 handler =====
|
|
133
|
+
file_handler = logging.FileHandler(log_file, encoding="utf-8")
|
|
134
|
+
file_handler.setLevel(logging.DEBUG)
|
|
135
|
+
file_handler.setFormatter(formatter)
|
|
136
|
+
|
|
137
|
+
# ===== 控制台 handler =====
|
|
138
|
+
console_handler = logging.StreamHandler(sys.stderr)
|
|
139
|
+
console_handler.setLevel(logging.INFO)
|
|
140
|
+
console_handler.setFormatter(formatter)
|
|
141
|
+
|
|
142
|
+
# ===== logger =====
|
|
143
|
+
logger = logging.getLogger("app")
|
|
144
|
+
logger.setLevel(logging.DEBUG)
|
|
145
|
+
logger.handlers.clear()
|
|
146
|
+
logger.addHandler(file_handler)
|
|
147
|
+
logger.addHandler(console_handler)
|
|
148
|
+
logger.propagate = False
|
|
149
|
+
|
|
150
|
+
flog = logger
|
|
151
|
+
clog = logger
|
|
152
|
+
|
|
153
|
+
# ===== stdout / stderr tee =====
|
|
154
|
+
original_stdout = sys.stdout
|
|
155
|
+
original_stderr = sys.stderr
|
|
156
|
+
|
|
157
|
+
sys.stdout = TeeStream(
|
|
158
|
+
original_stdout,
|
|
159
|
+
PrintToFile(file_handler, level=logging.INFO)
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
sys.stderr = TeeStream(
|
|
163
|
+
original_stderr,
|
|
164
|
+
PrintToFile(file_handler, level=logging.ERROR)
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# 确认文件已经创建
|
|
168
|
+
if not os.path.isfile(log_file):
|
|
169
|
+
raise RuntimeError(f"日志文件未创建: {log_file}")
|
|
93
170
|
|
|
94
|
-
def setLevel(log_level=logging.INFO):
|
|
95
|
-
flog.setLevel(log_level)
|
|
96
|
-
clog.setLevel(log_level)
|
ctools/util/jb_cut.py
CHANGED
ctools/web/bottle_web_base.py
CHANGED
|
@@ -207,6 +207,7 @@ def params_resolve(func):
|
|
|
207
207
|
dict_wrapper = DictWrapper({'body': params})
|
|
208
208
|
dict_wrapper.update(query_params.dict)
|
|
209
209
|
return func(params=auto_exchange(func, dict_wrapper), *args, **kwargs)
|
|
210
|
+
return None
|
|
210
211
|
else:
|
|
211
212
|
return func(*args, **kwargs)
|
|
212
213
|
|
ctools/web/bottle_webserver.py
CHANGED
|
@@ -19,7 +19,9 @@ from ctools.util.config_util import load_config
|
|
|
19
19
|
from ctools.web import bottle_web_base, bottle_webserver
|
|
20
20
|
from key_word_cloud.db_core.db_init import init_partitions
|
|
21
21
|
from patch_manager import patch_funcs
|
|
22
|
+
from ctools.web.ctoken import CToken
|
|
22
23
|
|
|
24
|
+
CToken.token_audience = 'server-name'
|
|
23
25
|
database.init_db('postgresql://postgres:123123@192.168.xx.xx:5432/xxx', default_schema='xxx', auto_gen_table=False, echo=False)
|
|
24
26
|
|
|
25
27
|
config = load_config('application.ini')
|
|
@@ -31,7 +33,7 @@ app = bottle_web_base.init_app("/api", True)
|
|
|
31
33
|
|
|
32
34
|
@bottle_web_base.before_intercept(0)
|
|
33
35
|
def token_check():
|
|
34
|
-
return bottle_web_base.common_auth_verify(
|
|
36
|
+
return bottle_web_base.common_auth_verify()
|
|
35
37
|
|
|
36
38
|
if __name__ == '__main__':
|
|
37
39
|
main_server = bottle_webserver.init_bottle(app)
|
|
@@ -41,7 +43,6 @@ if __name__ == '__main__':
|
|
|
41
43
|
|
|
42
44
|
_default_port = 8888
|
|
43
45
|
|
|
44
|
-
|
|
45
46
|
class CBottle:
|
|
46
47
|
|
|
47
48
|
def __init__(self, bottle: Bottle, port=_default_port, quiet=False):
|
|
@@ -56,6 +57,7 @@ class CBottle:
|
|
|
56
57
|
self.static_root = './static'
|
|
57
58
|
self.download_root = './download'
|
|
58
59
|
self.template_root = './templates'
|
|
60
|
+
self.server_path = '/api/'
|
|
59
61
|
|
|
60
62
|
@self.bottle.route(['/', '/index'])
|
|
61
63
|
def index():
|
|
@@ -120,33 +122,35 @@ class CBottle:
|
|
|
120
122
|
|
|
121
123
|
def run(self):
|
|
122
124
|
http_server = WSGIRefServer(port=self.port)
|
|
123
|
-
print('Click the link below to open the service homepage %s' % '\n \t\t http://localhost:%s \n \t\t http://%s:%s' % (self.port, sys_info.get_local_ipv4(), self.port), file=sys.stderr)
|
|
125
|
+
print('Click the link below to open the service homepage %s' % '\n \t\t http://localhost:%s \n \t\t http://%s:%s \n' % (self.port, sys_info.get_local_ipv4(), self.port), file=sys.stderr)
|
|
124
126
|
cache_white_list(self.bottle)
|
|
125
127
|
self.bottle.run(server=http_server, quiet=self.quiet)
|
|
126
128
|
|
|
127
129
|
def enable_spa_mode(self):
|
|
128
130
|
@self.bottle.error(404)
|
|
129
131
|
def error_404_handler(error):
|
|
130
|
-
if request.path.startswith(
|
|
132
|
+
if request.path.startswith(self.server_path):
|
|
131
133
|
response.status = 404
|
|
132
134
|
return R.error(resp=R.Code.cus_code(404, "资源未找到: {}".format(error.body)))
|
|
133
135
|
return static_file(filename=self.index_filename, root=self.index_root)
|
|
134
136
|
|
|
135
137
|
@self.bottle.error(401)
|
|
136
138
|
def unauthorized(error):
|
|
137
|
-
if request.path.startswith(
|
|
139
|
+
if request.path.startswith(self.server_path):
|
|
138
140
|
response.status = 401
|
|
139
141
|
return R.error(resp=R.Code.cus_code(401, "系统未授权! {}".format(error.body)))
|
|
140
142
|
response.status=301
|
|
141
143
|
response.set_header('Location', urljoin(request.url, '/'))
|
|
142
144
|
return response
|
|
143
145
|
|
|
144
|
-
def set_index(self, filename='index.html', root='./', is_tpl=False, redirect_url=None, spa=False, **kwargs):
|
|
146
|
+
def set_index(self, filename='index.html', root='./', is_tpl=False, redirect_url=None, spa=False, server_path="/api/", **kwargs):
|
|
145
147
|
self.index_root = root
|
|
146
148
|
self.index_filename = filename
|
|
147
149
|
self.is_tpl = is_tpl
|
|
148
150
|
self.redirect_url = redirect_url
|
|
149
|
-
if spa:
|
|
151
|
+
if spa:
|
|
152
|
+
self.server_path = server_path
|
|
153
|
+
self.enable_spa_mode()
|
|
150
154
|
self.tmp_args = kwargs
|
|
151
155
|
|
|
152
156
|
def set_static(self, root='./static'):
|
ctools/web/bottle_websocket.py
CHANGED
|
@@ -19,7 +19,7 @@ def get_ws_modules():
|
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
"""
|
|
22
|
-
ws_app = bottle_web_base.init_app('/websocket_demo')
|
|
22
|
+
ws_app = bottle_web_base.init_app('/websocket_demo', main_app=True)
|
|
23
23
|
|
|
24
24
|
@ws_app.route('/script_debug', apply=[websocket])
|
|
25
25
|
@bottle_web_base.rule('DOC:DOWNLOAD')
|
ctools/web/ctoken.py
CHANGED