hello-datap-component-base 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/PKG-INFO +63 -1
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/README.md +61 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/logger.py +26 -2
- hello_datap_component_base-0.2.2/hello_datap_component_base/oss_client.py +152 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/runner.py +67 -2
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base.egg-info/PKG-INFO +63 -1
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base.egg-info/SOURCES.txt +1 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base.egg-info/requires.txt +1 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/pyproject.toml +2 -1
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/__init__.py +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/base.py +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/cli.py +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/config.py +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/discover.py +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base/mns_client.py +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base.egg-info/dependency_links.txt +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base.egg-info/entry_points.txt +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/hello_datap_component_base.egg-info/top_level.txt +0 -0
- {hello_datap_component_base-0.2.0 → hello_datap_component_base-0.2.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hello-datap-component-base
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: A unified server management framework for data processing component
|
|
5
5
|
Author-email: zhaohaidong <zhaohaidong389@hellobike.com>
|
|
6
6
|
License: MIT
|
|
@@ -23,6 +23,7 @@ Requires-Dist: pydantic>=2.0.0
|
|
|
23
23
|
Requires-Dist: python-json-logger>=2.0.0
|
|
24
24
|
Requires-Dist: pyyaml>=6.0.0
|
|
25
25
|
Requires-Dist: aliyun-mns>=1.1.5
|
|
26
|
+
Requires-Dist: oss2>=2.18.0
|
|
26
27
|
Provides-Extra: dev
|
|
27
28
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
28
29
|
Requires-Dist: black>=23.0.0; extra == "dev"
|
|
@@ -173,6 +174,7 @@ result = await service.handle_request({"input": "data"})
|
|
|
173
174
|
- `params` 中的内容会作为输入数据传递给 `process` 方法
|
|
174
175
|
- `work_flow_id`、`work_flow_instance_id`、`task_id` 会包含在返回结果中
|
|
175
176
|
- 如果配置了 MNS 相关环境变量,处理结果会自动发送到 MNS 队列
|
|
177
|
+
- 如果配置了 OSS 相关环境变量,日志文件会自动上传到 OSS
|
|
176
178
|
|
|
177
179
|
#### HTTP远程配置
|
|
178
180
|
|
|
@@ -474,6 +476,41 @@ class YourService(BaseService):
|
|
|
474
476
|
- 仅对网络异常进行重试(如连接重置、超时等)
|
|
475
477
|
- 可通过环境变量自定义重试参数
|
|
476
478
|
|
|
479
|
+
### OSS 日志上传
|
|
480
|
+
|
|
481
|
+
如果配置了 OSS 相关环境变量,日志文件会自动上传到阿里云 OSS:
|
|
482
|
+
|
|
483
|
+
**配置方式:**
|
|
484
|
+
在配置文件的 `runtime_env.env_vars` 中设置:
|
|
485
|
+
```json
|
|
486
|
+
{
|
|
487
|
+
"runtime_env": {
|
|
488
|
+
"env_vars": {
|
|
489
|
+
"OSS_ENDPOINT": "https://oss-cn-shanghai.aliyuncs.com",
|
|
490
|
+
"OSS_ACCESS_KEY_ID": "your-access-key-id",
|
|
491
|
+
"OSS_ACCESS_KEY_SECRET": "your-access-key-secret",
|
|
492
|
+
"OSS_BUCKET_NAME": "your-bucket-name",
|
|
493
|
+
"OSS_LOG_PATH_PREFIX": "logs/service-name"
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
**功能特性:**
|
|
500
|
+
- ✅ 自动收集日志到文件(非 Ray 环境)
|
|
501
|
+
- ✅ 服务完成后自动上传日志文件到 OSS
|
|
502
|
+
- ✅ **上传成功后自动删除本地日志文件**,节省存储空间
|
|
503
|
+
- ✅ 日志文件路径格式:`logs/{service_name}/{task_id}_{timestamp}.log`
|
|
504
|
+
- ✅ **向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程
|
|
505
|
+
- ✅ 即使服务异常,也会尝试上传日志文件
|
|
506
|
+
|
|
507
|
+
**环境变量说明:**
|
|
508
|
+
- `OSS_ENDPOINT`(必需):OSS 服务端点,如 `https://oss-cn-shanghai.aliyuncs.com`
|
|
509
|
+
- `OSS_ACCESS_KEY_ID`(必需):OSS 访问密钥 ID
|
|
510
|
+
- `OSS_ACCESS_KEY_SECRET`(必需):OSS 访问密钥
|
|
511
|
+
- `OSS_BUCKET_NAME`(必需):OSS Bucket 名称
|
|
512
|
+
- `OSS_LOG_PATH_PREFIX`(可选):日志文件在 OSS 中的路径前缀,如 `logs/service-name`,默认为空
|
|
513
|
+
|
|
477
514
|
## 常见问题
|
|
478
515
|
|
|
479
516
|
**Q: 如何支持HTTP远程配置?**
|
|
@@ -579,6 +616,31 @@ A: 在配置文件的 `runtime_env.env_vars` 中设置 MNS 相关环境变量:
|
|
|
579
616
|
|
|
580
617
|
配置后,处理结果会自动发送到队列。如果未配置或发送失败,不会影响主流程。
|
|
581
618
|
|
|
619
|
+
**Q: 如何配置 OSS 日志上传?**
|
|
620
|
+
|
|
621
|
+
A: 在配置文件的 `runtime_env.env_vars` 中设置 OSS 相关环境变量:
|
|
622
|
+
```json
|
|
623
|
+
{
|
|
624
|
+
"runtime_env": {
|
|
625
|
+
"env_vars": {
|
|
626
|
+
"OSS_ENDPOINT": "https://oss-cn-shanghai.aliyuncs.com",
|
|
627
|
+
"OSS_ACCESS_KEY_ID": "your-access-key-id",
|
|
628
|
+
"OSS_ACCESS_KEY_SECRET": "your-access-key-secret",
|
|
629
|
+
"OSS_BUCKET_NAME": "your-bucket-name",
|
|
630
|
+
"OSS_LOG_PATH_PREFIX": "logs/service-name"
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
配置后,服务完成时会自动上传日志文件到 OSS。日志文件路径格式:`logs/{service_name}/{task_id}.log`。
|
|
637
|
+
|
|
638
|
+
**功能说明**:
|
|
639
|
+
- 上传成功后会自动删除本地日志文件,节省存储空间
|
|
640
|
+
- 如果上传失败,本地日志文件会保留,便于排查问题
|
|
641
|
+
|
|
642
|
+
**向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程。
|
|
643
|
+
|
|
582
644
|
**Q: 返回结果格式是什么?**
|
|
583
645
|
|
|
584
646
|
A: `handle_request` 方法会自动封装返回结果:
|
|
@@ -143,6 +143,7 @@ result = await service.handle_request({"input": "data"})
|
|
|
143
143
|
- `params` 中的内容会作为输入数据传递给 `process` 方法
|
|
144
144
|
- `work_flow_id`、`work_flow_instance_id`、`task_id` 会包含在返回结果中
|
|
145
145
|
- 如果配置了 MNS 相关环境变量,处理结果会自动发送到 MNS 队列
|
|
146
|
+
- 如果配置了 OSS 相关环境变量,日志文件会自动上传到 OSS
|
|
146
147
|
|
|
147
148
|
#### HTTP远程配置
|
|
148
149
|
|
|
@@ -444,6 +445,41 @@ class YourService(BaseService):
|
|
|
444
445
|
- 仅对网络异常进行重试(如连接重置、超时等)
|
|
445
446
|
- 可通过环境变量自定义重试参数
|
|
446
447
|
|
|
448
|
+
### OSS 日志上传
|
|
449
|
+
|
|
450
|
+
如果配置了 OSS 相关环境变量,日志文件会自动上传到阿里云 OSS:
|
|
451
|
+
|
|
452
|
+
**配置方式:**
|
|
453
|
+
在配置文件的 `runtime_env.env_vars` 中设置:
|
|
454
|
+
```json
|
|
455
|
+
{
|
|
456
|
+
"runtime_env": {
|
|
457
|
+
"env_vars": {
|
|
458
|
+
"OSS_ENDPOINT": "https://oss-cn-shanghai.aliyuncs.com",
|
|
459
|
+
"OSS_ACCESS_KEY_ID": "your-access-key-id",
|
|
460
|
+
"OSS_ACCESS_KEY_SECRET": "your-access-key-secret",
|
|
461
|
+
"OSS_BUCKET_NAME": "your-bucket-name",
|
|
462
|
+
"OSS_LOG_PATH_PREFIX": "logs/service-name"
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
**功能特性:**
|
|
469
|
+
- ✅ 自动收集日志到文件(非 Ray 环境)
|
|
470
|
+
- ✅ 服务完成后自动上传日志文件到 OSS
|
|
471
|
+
- ✅ **上传成功后自动删除本地日志文件**,节省存储空间
|
|
472
|
+
- ✅ 日志文件路径格式:`logs/{service_name}/{task_id}_{timestamp}.log`
|
|
473
|
+
- ✅ **向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程
|
|
474
|
+
- ✅ 即使服务异常,也会尝试上传日志文件
|
|
475
|
+
|
|
476
|
+
**环境变量说明:**
|
|
477
|
+
- `OSS_ENDPOINT`(必需):OSS 服务端点,如 `https://oss-cn-shanghai.aliyuncs.com`
|
|
478
|
+
- `OSS_ACCESS_KEY_ID`(必需):OSS 访问密钥 ID
|
|
479
|
+
- `OSS_ACCESS_KEY_SECRET`(必需):OSS 访问密钥
|
|
480
|
+
- `OSS_BUCKET_NAME`(必需):OSS Bucket 名称
|
|
481
|
+
- `OSS_LOG_PATH_PREFIX`(可选):日志文件在 OSS 中的路径前缀,如 `logs/service-name`,默认为空
|
|
482
|
+
|
|
447
483
|
## 常见问题
|
|
448
484
|
|
|
449
485
|
**Q: 如何支持HTTP远程配置?**
|
|
@@ -549,6 +585,31 @@ A: 在配置文件的 `runtime_env.env_vars` 中设置 MNS 相关环境变量:
|
|
|
549
585
|
|
|
550
586
|
配置后,处理结果会自动发送到队列。如果未配置或发送失败,不会影响主流程。
|
|
551
587
|
|
|
588
|
+
**Q: 如何配置 OSS 日志上传?**
|
|
589
|
+
|
|
590
|
+
A: 在配置文件的 `runtime_env.env_vars` 中设置 OSS 相关环境变量:
|
|
591
|
+
```json
|
|
592
|
+
{
|
|
593
|
+
"runtime_env": {
|
|
594
|
+
"env_vars": {
|
|
595
|
+
"OSS_ENDPOINT": "https://oss-cn-shanghai.aliyuncs.com",
|
|
596
|
+
"OSS_ACCESS_KEY_ID": "your-access-key-id",
|
|
597
|
+
"OSS_ACCESS_KEY_SECRET": "your-access-key-secret",
|
|
598
|
+
"OSS_BUCKET_NAME": "your-bucket-name",
|
|
599
|
+
"OSS_LOG_PATH_PREFIX": "logs/service-name"
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
配置后,服务完成时会自动上传日志文件到 OSS。日志文件路径格式:`logs/{service_name}/{task_id}.log`。
|
|
606
|
+
|
|
607
|
+
**功能说明**:
|
|
608
|
+
- 上传成功后会自动删除本地日志文件,节省存储空间
|
|
609
|
+
- 如果上传失败,本地日志文件会保留,便于排查问题
|
|
610
|
+
|
|
611
|
+
**向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程。
|
|
612
|
+
|
|
552
613
|
**Q: 返回结果格式是什么?**
|
|
553
614
|
|
|
554
615
|
A: `handle_request` 方法会自动封装返回结果:
|
|
@@ -58,12 +58,26 @@ class CustomJsonFormatter(jsonlogger.JsonFormatter):
|
|
|
58
58
|
log_record['version'] = log_record['version']
|
|
59
59
|
|
|
60
60
|
|
|
61
|
+
# 全局变量:存储日志文件路径
|
|
62
|
+
_log_file_path: Optional[str] = None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def get_log_file_path() -> Optional[str]:
|
|
66
|
+
"""
|
|
67
|
+
获取当前日志文件路径
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
日志文件路径,如果未创建日志文件则返回 None
|
|
71
|
+
"""
|
|
72
|
+
return _log_file_path
|
|
73
|
+
|
|
74
|
+
|
|
61
75
|
def setup_logging(
|
|
62
76
|
level: str = "INFO",
|
|
63
77
|
json_format: bool = False,
|
|
64
78
|
service_name: str = "unknown",
|
|
65
79
|
version: Optional[str] = None
|
|
66
|
-
):
|
|
80
|
+
) -> Optional[str]:
|
|
67
81
|
"""
|
|
68
82
|
设置日志配置
|
|
69
83
|
|
|
@@ -74,7 +88,12 @@ def setup_logging(
|
|
|
74
88
|
json_format: 是否使用 JSON 格式
|
|
75
89
|
service_name: 服务名称
|
|
76
90
|
version: 服务版本(可选)
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
日志文件路径,如果未创建日志文件则返回 None
|
|
77
94
|
"""
|
|
95
|
+
global _log_file_path
|
|
96
|
+
|
|
78
97
|
# 检测是否在 Ray 环境中运行
|
|
79
98
|
is_ray_env = False
|
|
80
99
|
try:
|
|
@@ -125,14 +144,19 @@ def setup_logging(
|
|
|
125
144
|
root_logger.addHandler(error_handler)
|
|
126
145
|
|
|
127
146
|
# 文件处理器(在 Ray 环境中可能不可用,优雅处理)
|
|
147
|
+
log_file_path = None
|
|
128
148
|
if not is_ray_env:
|
|
129
149
|
# 非 Ray 环境才尝试创建文件日志
|
|
130
150
|
try:
|
|
131
|
-
|
|
151
|
+
log_file_path = f"{service_name}.log"
|
|
152
|
+
file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
|
|
132
153
|
file_handler.setFormatter(formatter)
|
|
133
154
|
root_logger.addHandler(file_handler)
|
|
155
|
+
_log_file_path = log_file_path
|
|
134
156
|
except (IOError, PermissionError):
|
|
135
157
|
pass # 无法创建日志文件,只输出到控制台
|
|
158
|
+
|
|
159
|
+
return log_file_path
|
|
136
160
|
|
|
137
161
|
|
|
138
162
|
def get_service_logger(name: str, version: Optional[str] = None) -> ServiceLoggerAdapter:
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""
|
|
2
|
+
阿里云 OSS(对象存储)客户端
|
|
3
|
+
用于上传日志文件到 OSS
|
|
4
|
+
"""
|
|
5
|
+
import os
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Optional
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class OSSClient:
|
|
14
|
+
"""阿里云 OSS 客户端"""
|
|
15
|
+
|
|
16
|
+
def __init__(self):
|
|
17
|
+
"""初始化 OSS 客户端(延迟加载)"""
|
|
18
|
+
self._client = None
|
|
19
|
+
self._bucket = None
|
|
20
|
+
self._initialized = False
|
|
21
|
+
|
|
22
|
+
def _init_client(self):
|
|
23
|
+
"""初始化 OSS 客户端(延迟加载)"""
|
|
24
|
+
if self._initialized:
|
|
25
|
+
return
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
# 尝试导入阿里云 OSS SDK
|
|
29
|
+
import oss2
|
|
30
|
+
|
|
31
|
+
# 从环境变量获取 OSS 配置
|
|
32
|
+
endpoint = os.environ.get('OSS_ENDPOINT_FOR_LOG')
|
|
33
|
+
access_key_id = os.environ.get('OSS_ACCESS_KEY_ID_FOR_LOG')
|
|
34
|
+
access_key_secret = os.environ.get('OSS_ACCESS_KEY_SECRET_FOR_LOG')
|
|
35
|
+
bucket_name = os.environ.get('OSS_BUCKET_NAME_FOR_LOG')
|
|
36
|
+
|
|
37
|
+
# 如果 OSS_ENDPOINT 不存在,静默跳过(向下兼容)
|
|
38
|
+
if not endpoint:
|
|
39
|
+
self._initialized = True
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
# 如果 OSS_ENDPOINT 存在但其他配置不完整,记录警告
|
|
43
|
+
if not all([access_key_id, access_key_secret, bucket_name]):
|
|
44
|
+
logger.warning(
|
|
45
|
+
"OSS 配置不完整,无法上传日志文件。"
|
|
46
|
+
"需要设置环境变量: OSS_ACCESS_KEY_ID_FOR_LOG, OSS_ACCESS_KEY_SECRET_FOR_LOG, OSS_BUCKET_NAME_FOR_LOG"
|
|
47
|
+
)
|
|
48
|
+
self._initialized = True
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
# 创建 OSS 认证对象
|
|
52
|
+
auth = oss2.Auth(access_key_id, access_key_secret)
|
|
53
|
+
|
|
54
|
+
# 创建 Bucket 对象
|
|
55
|
+
self._bucket = oss2.Bucket(auth, endpoint, bucket_name)
|
|
56
|
+
self._client = auth
|
|
57
|
+
self._initialized = True
|
|
58
|
+
logger.info(f"OSS 客户端初始化成功,Bucket: {bucket_name}")
|
|
59
|
+
|
|
60
|
+
except ImportError:
|
|
61
|
+
logger.warning(
|
|
62
|
+
"未安装阿里云 OSS SDK,无法上传日志文件。"
|
|
63
|
+
"请安装: pip install oss2"
|
|
64
|
+
)
|
|
65
|
+
self._initialized = True
|
|
66
|
+
except Exception as e:
|
|
67
|
+
logger.error(f"初始化 OSS 客户端失败: {e}")
|
|
68
|
+
self._initialized = True
|
|
69
|
+
|
|
70
|
+
def upload_file(
|
|
71
|
+
self,
|
|
72
|
+
local_file_path: str,
|
|
73
|
+
oss_object_key: Optional[str] = None
|
|
74
|
+
) -> bool:
|
|
75
|
+
"""
|
|
76
|
+
上传文件到 OSS
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
local_file_path: 本地文件路径
|
|
80
|
+
oss_object_key: OSS 对象键(路径),如果为 None 则使用文件名
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
是否上传成功
|
|
84
|
+
"""
|
|
85
|
+
self._init_client()
|
|
86
|
+
|
|
87
|
+
# 如果未配置 OSS,静默跳过(向下兼容)
|
|
88
|
+
if not self._bucket:
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
# 检查文件是否存在
|
|
92
|
+
file_path = Path(local_file_path)
|
|
93
|
+
if not file_path.exists():
|
|
94
|
+
logger.warning(f"日志文件不存在: {local_file_path}")
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
try:
|
|
98
|
+
# 如果没有指定 OSS 对象键,使用文件名
|
|
99
|
+
if oss_object_key is None:
|
|
100
|
+
oss_object_key = file_path.name
|
|
101
|
+
|
|
102
|
+
# 从环境变量获取 OSS 路径前缀(可选)
|
|
103
|
+
oss_path_prefix = os.environ.get('OSS_LOG_PATH_PREFIX', '')
|
|
104
|
+
if oss_path_prefix:
|
|
105
|
+
# 确保路径前缀以 / 结尾(如果没有)
|
|
106
|
+
if not oss_path_prefix.endswith('/'):
|
|
107
|
+
oss_path_prefix += '/'
|
|
108
|
+
oss_object_key = oss_path_prefix + oss_object_key
|
|
109
|
+
|
|
110
|
+
# 上传文件
|
|
111
|
+
with open(local_file_path, 'rb') as f:
|
|
112
|
+
self._bucket.put_object(oss_object_key, f)
|
|
113
|
+
|
|
114
|
+
logger.info(f"日志文件已上传到 OSS: {oss_object_key}")
|
|
115
|
+
return True
|
|
116
|
+
|
|
117
|
+
except Exception as e:
|
|
118
|
+
logger.error(f"上传日志文件到 OSS 失败: {e}", exc_info=True)
|
|
119
|
+
return False
|
|
120
|
+
|
|
121
|
+
def upload_log_file(self, log_file_path: str) -> bool:
|
|
122
|
+
"""
|
|
123
|
+
上传日志文件到 OSS(便捷方法)
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
log_file_path: 日志文件路径
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
是否上传成功
|
|
130
|
+
"""
|
|
131
|
+
return self.upload_file(log_file_path)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def get_oss_client() -> Optional[OSSClient]:
|
|
135
|
+
"""
|
|
136
|
+
获取 OSS 客户端实例
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
OSSClient 实例,如果配置不完整则返回 None
|
|
140
|
+
"""
|
|
141
|
+
# 如果环境变量中没有 OSS_ENDPOINT 配置,则不使用 OSS,直接返回 None(向下兼容)
|
|
142
|
+
if not os.environ.get('OSS_ENDPOINT_FOR_LOG'):
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
client = OSSClient()
|
|
146
|
+
client._init_client()
|
|
147
|
+
|
|
148
|
+
# 如果客户端未正确初始化,返回 None
|
|
149
|
+
if not client._bucket:
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
return client
|
|
@@ -43,7 +43,8 @@ class ServiceRunner:
|
|
|
43
43
|
def setup_environment(self):
|
|
44
44
|
"""设置运行环境"""
|
|
45
45
|
# 设置日志
|
|
46
|
-
setup_logging
|
|
46
|
+
from .logger import setup_logging, get_log_file_path
|
|
47
|
+
self.log_file_path = setup_logging(
|
|
47
48
|
level=self.config.runtime_env.env_vars.get("LOG_LEVEL", "INFO")
|
|
48
49
|
if self.config.runtime_env and self.config.runtime_env.env_vars
|
|
49
50
|
else "INFO",
|
|
@@ -123,7 +124,9 @@ class ServiceRunner:
|
|
|
123
124
|
print(f"Starting service: {self.config.name}")
|
|
124
125
|
if self.config.version:
|
|
125
126
|
print(f"Version: {self.config.version}")
|
|
126
|
-
|
|
127
|
+
import json
|
|
128
|
+
params_to_print = self.config.params if self.config.params is not None else {}
|
|
129
|
+
print(f"Param: {json.dumps(params_to_print, indent=2, ensure_ascii=False)}")
|
|
127
130
|
print("\n" + "=" * 60)
|
|
128
131
|
print("Processing request...")
|
|
129
132
|
print("=" * 60 + "\n")
|
|
@@ -149,6 +152,9 @@ class ServiceRunner:
|
|
|
149
152
|
# 发送结果到 MNS 队列
|
|
150
153
|
self._send_result_to_mns(result)
|
|
151
154
|
|
|
155
|
+
# 上传日志文件到 OSS(如果配置了)
|
|
156
|
+
self._upload_log_to_oss()
|
|
157
|
+
|
|
152
158
|
print("Service completed successfully.")
|
|
153
159
|
print("=" * 60)
|
|
154
160
|
|
|
@@ -173,6 +179,12 @@ class ServiceRunner:
|
|
|
173
179
|
except Exception as mns_error:
|
|
174
180
|
print(f"Failed to send error result to MNS: {mns_error}", file=sys.stderr)
|
|
175
181
|
|
|
182
|
+
# 即使发生异常,也要尝试上传日志文件到 OSS
|
|
183
|
+
try:
|
|
184
|
+
self._upload_log_to_oss()
|
|
185
|
+
except Exception as oss_error:
|
|
186
|
+
print(f"Failed to upload log to OSS: {oss_error}", file=sys.stderr)
|
|
187
|
+
|
|
176
188
|
raise
|
|
177
189
|
|
|
178
190
|
def run(self):
|
|
@@ -225,6 +237,59 @@ class ServiceRunner:
|
|
|
225
237
|
# 如果 client 为 None(配置不完整),静默跳过,不打印警告
|
|
226
238
|
except Exception as e:
|
|
227
239
|
print(f"⚠️ Error sending result to MNS queue: {e}", file=sys.stderr)
|
|
240
|
+
|
|
241
|
+
def _upload_log_to_oss(self):
|
|
242
|
+
"""
|
|
243
|
+
上传日志文件到 OSS
|
|
244
|
+
|
|
245
|
+
如果未配置 OSS 环境变量,则静默跳过(向下兼容)
|
|
246
|
+
"""
|
|
247
|
+
try:
|
|
248
|
+
from .oss_client import get_oss_client
|
|
249
|
+
from .logger import get_log_file_path
|
|
250
|
+
import os
|
|
251
|
+
|
|
252
|
+
# 如果环境变量中没有 OSS_ENDPOINT 配置,则不上传,静默跳过(向下兼容)
|
|
253
|
+
if not os.environ.get('OSS_ENDPOINT'):
|
|
254
|
+
return
|
|
255
|
+
|
|
256
|
+
# 获取日志文件路径
|
|
257
|
+
log_file_path = getattr(self, 'log_file_path', None)
|
|
258
|
+
if not log_file_path:
|
|
259
|
+
# 如果 runner 中没有保存路径,尝试从 logger 获取
|
|
260
|
+
log_file_path = get_log_file_path()
|
|
261
|
+
|
|
262
|
+
if not log_file_path:
|
|
263
|
+
# 没有日志文件,静默跳过
|
|
264
|
+
return
|
|
265
|
+
|
|
266
|
+
client = get_oss_client()
|
|
267
|
+
if client:
|
|
268
|
+
# 生成 OSS 对象键(可选:包含时间戳和服务信息)
|
|
269
|
+
from datetime import datetime
|
|
270
|
+
import os
|
|
271
|
+
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
272
|
+
service_name = self.config.name
|
|
273
|
+
task_id = self.config.task_id or 'unknown'
|
|
274
|
+
|
|
275
|
+
# 生成 OSS 对象键:logs/{service_name}/{task_id}_{timestamp}.log
|
|
276
|
+
oss_object_key = f"logs/{service_name}/{task_id}.log"
|
|
277
|
+
|
|
278
|
+
success = client.upload_file(log_file_path, oss_object_key)
|
|
279
|
+
if success:
|
|
280
|
+
print(f"✅ Log file uploaded to OSS: {oss_object_key}")
|
|
281
|
+
# 上传成功后删除本地日志文件
|
|
282
|
+
try:
|
|
283
|
+
if os.path.exists(log_file_path):
|
|
284
|
+
os.remove(log_file_path)
|
|
285
|
+
print(f"✅ Local log file deleted: {log_file_path}")
|
|
286
|
+
except Exception as delete_error:
|
|
287
|
+
print(f"⚠️ Failed to delete local log file: {delete_error}", file=sys.stderr)
|
|
288
|
+
else:
|
|
289
|
+
print("⚠️ Failed to upload log file to OSS")
|
|
290
|
+
# 如果 client 为 None(配置不完整),静默跳过,不打印警告
|
|
291
|
+
except Exception as e:
|
|
292
|
+
print(f"⚠️ Error uploading log to OSS: {e}", file=sys.stderr)
|
|
228
293
|
|
|
229
294
|
async def process_request(self, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
230
295
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hello-datap-component-base
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: A unified server management framework for data processing component
|
|
5
5
|
Author-email: zhaohaidong <zhaohaidong389@hellobike.com>
|
|
6
6
|
License: MIT
|
|
@@ -23,6 +23,7 @@ Requires-Dist: pydantic>=2.0.0
|
|
|
23
23
|
Requires-Dist: python-json-logger>=2.0.0
|
|
24
24
|
Requires-Dist: pyyaml>=6.0.0
|
|
25
25
|
Requires-Dist: aliyun-mns>=1.1.5
|
|
26
|
+
Requires-Dist: oss2>=2.18.0
|
|
26
27
|
Provides-Extra: dev
|
|
27
28
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
28
29
|
Requires-Dist: black>=23.0.0; extra == "dev"
|
|
@@ -173,6 +174,7 @@ result = await service.handle_request({"input": "data"})
|
|
|
173
174
|
- `params` 中的内容会作为输入数据传递给 `process` 方法
|
|
174
175
|
- `work_flow_id`、`work_flow_instance_id`、`task_id` 会包含在返回结果中
|
|
175
176
|
- 如果配置了 MNS 相关环境变量,处理结果会自动发送到 MNS 队列
|
|
177
|
+
- 如果配置了 OSS 相关环境变量,日志文件会自动上传到 OSS
|
|
176
178
|
|
|
177
179
|
#### HTTP远程配置
|
|
178
180
|
|
|
@@ -474,6 +476,41 @@ class YourService(BaseService):
|
|
|
474
476
|
- 仅对网络异常进行重试(如连接重置、超时等)
|
|
475
477
|
- 可通过环境变量自定义重试参数
|
|
476
478
|
|
|
479
|
+
### OSS 日志上传
|
|
480
|
+
|
|
481
|
+
如果配置了 OSS 相关环境变量,日志文件会自动上传到阿里云 OSS:
|
|
482
|
+
|
|
483
|
+
**配置方式:**
|
|
484
|
+
在配置文件的 `runtime_env.env_vars` 中设置:
|
|
485
|
+
```json
|
|
486
|
+
{
|
|
487
|
+
"runtime_env": {
|
|
488
|
+
"env_vars": {
|
|
489
|
+
"OSS_ENDPOINT": "https://oss-cn-shanghai.aliyuncs.com",
|
|
490
|
+
"OSS_ACCESS_KEY_ID": "your-access-key-id",
|
|
491
|
+
"OSS_ACCESS_KEY_SECRET": "your-access-key-secret",
|
|
492
|
+
"OSS_BUCKET_NAME": "your-bucket-name",
|
|
493
|
+
"OSS_LOG_PATH_PREFIX": "logs/service-name"
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
**功能特性:**
|
|
500
|
+
- ✅ 自动收集日志到文件(非 Ray 环境)
|
|
501
|
+
- ✅ 服务完成后自动上传日志文件到 OSS
|
|
502
|
+
- ✅ **上传成功后自动删除本地日志文件**,节省存储空间
|
|
503
|
+
- ✅ 日志文件路径格式:`logs/{service_name}/{task_id}_{timestamp}.log`
|
|
504
|
+
- ✅ **向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程
|
|
505
|
+
- ✅ 即使服务异常,也会尝试上传日志文件
|
|
506
|
+
|
|
507
|
+
**环境变量说明:**
|
|
508
|
+
- `OSS_ENDPOINT`(必需):OSS 服务端点,如 `https://oss-cn-shanghai.aliyuncs.com`
|
|
509
|
+
- `OSS_ACCESS_KEY_ID`(必需):OSS 访问密钥 ID
|
|
510
|
+
- `OSS_ACCESS_KEY_SECRET`(必需):OSS 访问密钥
|
|
511
|
+
- `OSS_BUCKET_NAME`(必需):OSS Bucket 名称
|
|
512
|
+
- `OSS_LOG_PATH_PREFIX`(可选):日志文件在 OSS 中的路径前缀,如 `logs/service-name`,默认为空
|
|
513
|
+
|
|
477
514
|
## 常见问题
|
|
478
515
|
|
|
479
516
|
**Q: 如何支持HTTP远程配置?**
|
|
@@ -579,6 +616,31 @@ A: 在配置文件的 `runtime_env.env_vars` 中设置 MNS 相关环境变量:
|
|
|
579
616
|
|
|
580
617
|
配置后,处理结果会自动发送到队列。如果未配置或发送失败,不会影响主流程。
|
|
581
618
|
|
|
619
|
+
**Q: 如何配置 OSS 日志上传?**
|
|
620
|
+
|
|
621
|
+
A: 在配置文件的 `runtime_env.env_vars` 中设置 OSS 相关环境变量:
|
|
622
|
+
```json
|
|
623
|
+
{
|
|
624
|
+
"runtime_env": {
|
|
625
|
+
"env_vars": {
|
|
626
|
+
"OSS_ENDPOINT": "https://oss-cn-shanghai.aliyuncs.com",
|
|
627
|
+
"OSS_ACCESS_KEY_ID": "your-access-key-id",
|
|
628
|
+
"OSS_ACCESS_KEY_SECRET": "your-access-key-secret",
|
|
629
|
+
"OSS_BUCKET_NAME": "your-bucket-name",
|
|
630
|
+
"OSS_LOG_PATH_PREFIX": "logs/service-name"
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
配置后,服务完成时会自动上传日志文件到 OSS。日志文件路径格式:`logs/{service_name}/{task_id}.log`。
|
|
637
|
+
|
|
638
|
+
**功能说明**:
|
|
639
|
+
- 上传成功后会自动删除本地日志文件,节省存储空间
|
|
640
|
+
- 如果上传失败,本地日志文件会保留,便于排查问题
|
|
641
|
+
|
|
642
|
+
**向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程。
|
|
643
|
+
|
|
582
644
|
**Q: 返回结果格式是什么?**
|
|
583
645
|
|
|
584
646
|
A: `handle_request` 方法会自动封装返回结果:
|
|
@@ -7,6 +7,7 @@ hello_datap_component_base/config.py
|
|
|
7
7
|
hello_datap_component_base/discover.py
|
|
8
8
|
hello_datap_component_base/logger.py
|
|
9
9
|
hello_datap_component_base/mns_client.py
|
|
10
|
+
hello_datap_component_base/oss_client.py
|
|
10
11
|
hello_datap_component_base/runner.py
|
|
11
12
|
hello_datap_component_base.egg-info/PKG-INFO
|
|
12
13
|
hello_datap_component_base.egg-info/SOURCES.txt
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hello-datap-component-base"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "A unified server management framework for data processing component"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.8"
|
|
@@ -33,6 +33,7 @@ dependencies = [
|
|
|
33
33
|
"python-json-logger>=2.0.0",
|
|
34
34
|
"pyyaml>=6.0.0",
|
|
35
35
|
"aliyun-mns>=1.1.5",
|
|
36
|
+
"oss2>=2.18.0",
|
|
36
37
|
]
|
|
37
38
|
|
|
38
39
|
[project.optional-dependencies]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|