hello-datap-component-base 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hello_datap_component_base/__init__.py +8 -3
- hello_datap_component_base/cli.py +18 -2
- hello_datap_component_base/data/__init__.py +12 -0
- hello_datap_component_base/data/bag_data_service.py +422 -0
- hello_datap_component_base/discover.py +117 -3
- {hello_datap_component_base-0.2.3.dist-info → hello_datap_component_base-0.2.5.dist-info}/METADATA +103 -6
- hello_datap_component_base-0.2.5.dist-info/RECORD +16 -0
- hello_datap_component_base-0.2.3.dist-info/RECORD +0 -14
- {hello_datap_component_base-0.2.3.dist-info → hello_datap_component_base-0.2.5.dist-info}/WHEEL +0 -0
- {hello_datap_component_base-0.2.3.dist-info → hello_datap_component_base-0.2.5.dist-info}/entry_points.txt +0 -0
- {hello_datap_component_base-0.2.3.dist-info → hello_datap_component_base-0.2.5.dist-info}/top_level.txt +0 -0
|
@@ -13,9 +13,11 @@ from .discover import find_service_classes, get_single_service_class
|
|
|
13
13
|
# 导入 logger 实例
|
|
14
14
|
from .logger import logger
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
# 导入数据服务
|
|
17
|
+
from .data import BagDataService, BagJsonResult
|
|
18
|
+
|
|
19
|
+
__version__ = "0.2.5"
|
|
20
|
+
__author__ = "Data Processing Team"
|
|
19
21
|
|
|
20
22
|
__all__ = [
|
|
21
23
|
"BaseService",
|
|
@@ -28,4 +30,7 @@ __all__ = [
|
|
|
28
30
|
"logger",
|
|
29
31
|
"find_service_classes",
|
|
30
32
|
"get_single_service_class",
|
|
33
|
+
# 数据服务
|
|
34
|
+
"BagDataService",
|
|
35
|
+
"BagJsonResult",
|
|
31
36
|
]
|
|
@@ -11,7 +11,7 @@ from .config import ServerConfig
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
@click.group()
|
|
14
|
-
@click.version_option(version="0.2.
|
|
14
|
+
@click.version_option(version="0.2.4")
|
|
15
15
|
def cli():
|
|
16
16
|
"""数据处理平台组件基类 - 统一的服务管理框架"""
|
|
17
17
|
pass
|
|
@@ -19,13 +19,29 @@ def cli():
|
|
|
19
19
|
|
|
20
20
|
@cli.command()
|
|
21
21
|
@click.argument("config_path")
|
|
22
|
-
@click.option(
|
|
22
|
+
@click.option(
|
|
23
|
+
"--class-name", "-c",
|
|
24
|
+
help="指定要使用的服务类名或完整路径。支持格式: ClassName, module.ClassName, path/to/module.ClassName"
|
|
25
|
+
)
|
|
23
26
|
def start(config_path: str, class_name: Optional[str] = None):
|
|
24
27
|
"""
|
|
25
28
|
启动服务并执行一次处理(支持本地文件路径或HTTP URL)
|
|
26
29
|
|
|
27
30
|
输入数据从配置文件的 params.input_data 中获取。
|
|
28
31
|
如果 params.input_data 不存在,将使用默认测试数据。
|
|
32
|
+
|
|
33
|
+
\b
|
|
34
|
+
--class-name 支持的格式:
|
|
35
|
+
- ClassName 仅类名
|
|
36
|
+
- module.ClassName 模块名.类名
|
|
37
|
+
- path/to/module.ClassName 相对路径/模块名.类名
|
|
38
|
+
- path.to.module.ClassName 点分隔的完整路径
|
|
39
|
+
|
|
40
|
+
\b
|
|
41
|
+
示例:
|
|
42
|
+
component_manager start config.json -c MyService
|
|
43
|
+
component_manager start config.json -c example_service.MyService
|
|
44
|
+
component_manager start config.json -c services/my_service.MyService
|
|
29
45
|
"""
|
|
30
46
|
runner = ServiceRunner(config_path, class_name)
|
|
31
47
|
runner.run()
|
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""
|
|
2
|
+
BagData 服务 - 获取 Bag 数据的 OSS 地址
|
|
3
|
+
|
|
4
|
+
环境变量配置:
|
|
5
|
+
- BAG_DATA_APP_KEY: API 密钥(必需)
|
|
6
|
+
- BAG_DATA_API_URL: API 地址(必需)
|
|
7
|
+
- SKIP_SSL_VERIFY: 是否跳过 SSL 验证(可选,默认 false)
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
from typing import Optional
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
import requests
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _get_env_or_raise(key: str, default: Optional[str] = None) -> str:
|
|
22
|
+
"""获取环境变量,如果未设置且无默认值则抛出异常"""
|
|
23
|
+
value = os.environ.get(key, default)
|
|
24
|
+
if value is None:
|
|
25
|
+
raise ValueError(
|
|
26
|
+
f"环境变量 {key} 未设置。请设置该环境变量后重试。\n"
|
|
27
|
+
f"示例: export {key}=your_value"
|
|
28
|
+
)
|
|
29
|
+
return value
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# 从环境变量获取配置(延迟加载,在实际使用时才读取)
|
|
33
|
+
def _get_app_key() -> str:
|
|
34
|
+
"""获取 API 密钥"""
|
|
35
|
+
return _get_env_or_raise("BAG_DATA_APP_KEY")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _get_api_url() -> str:
|
|
39
|
+
"""获取 API 地址"""
|
|
40
|
+
return _get_env_or_raise("BAG_DATA_API_URL")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# 目标 OSS 配置(固定值)
|
|
44
|
+
TARGET_OSS_BUCKET = "infra-hads-artifacts"
|
|
45
|
+
TARGET_OSS_PREFIX = "bag_essential"
|
|
46
|
+
|
|
47
|
+
# JSON 文件类型(相对路径,不含敏感信息)
|
|
48
|
+
JSON_TYPES = {
|
|
49
|
+
"camera_forward_wide": "camera_forward_wide/camera_forward_wide.json",
|
|
50
|
+
"localization": "localization/localization.json",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# SSL 验证配置
|
|
55
|
+
def _skip_ssl_verify() -> bool:
|
|
56
|
+
"""是否跳过 SSL 验证"""
|
|
57
|
+
return os.environ.get("SKIP_SSL_VERIFY", "").lower() in ("true", "1", "yes")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class BagJsonResult:
|
|
62
|
+
"""Bag JSON 结果 - 只包含 OSS 地址"""
|
|
63
|
+
bag_name: str
|
|
64
|
+
camera_forward_wide_url: Optional[str] = None
|
|
65
|
+
localization_url: Optional[str] = None
|
|
66
|
+
errors: Optional[list] = None
|
|
67
|
+
|
|
68
|
+
def to_dict(self) -> dict:
|
|
69
|
+
return {
|
|
70
|
+
"bag_name": self.bag_name,
|
|
71
|
+
"camera_forward_wide_url": self.camera_forward_wide_url,
|
|
72
|
+
"localization_url": self.localization_url,
|
|
73
|
+
"errors": self.errors,
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class BagDataService:
|
|
78
|
+
"""
|
|
79
|
+
Bag 数据服务
|
|
80
|
+
|
|
81
|
+
用于根据 bag_name 获取对应的 JSON 文件 OSS 地址
|
|
82
|
+
|
|
83
|
+
环境变量配置:
|
|
84
|
+
- BAG_DATA_APP_KEY: API 密钥(必需)
|
|
85
|
+
- BAG_DATA_API_URL: API 地址(必需)
|
|
86
|
+
- SKIP_SSL_VERIFY: 是否跳过 SSL 验证(可选,默认 false)
|
|
87
|
+
|
|
88
|
+
使用示例:
|
|
89
|
+
# 首先设置环境变量
|
|
90
|
+
# export BAG_DATA_APP_KEY=your_app_key
|
|
91
|
+
# export BAG_DATA_API_URL=https://your-api-url.com
|
|
92
|
+
|
|
93
|
+
from hello_datap_component_base.data import BagDataService
|
|
94
|
+
|
|
95
|
+
service = BagDataService()
|
|
96
|
+
|
|
97
|
+
# 获取单个 bag 的 OSS 地址
|
|
98
|
+
result = service.get_bag_json("2_033_20260123-225631_0")
|
|
99
|
+
print(result.camera_forward_wide_url) # OSS 地址
|
|
100
|
+
print(result.localization_url) # OSS 地址
|
|
101
|
+
|
|
102
|
+
# 批量获取
|
|
103
|
+
results = service.get_bag_json_batch(["bag1", "bag2"])
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
def __init__(self, app_key: Optional[str] = None, api_url: Optional[str] = None):
|
|
107
|
+
"""
|
|
108
|
+
初始化服务
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
app_key: API 密钥(可选,默认从环境变量 BAG_DATA_APP_KEY 获取)
|
|
112
|
+
api_url: API 地址(可选,默认从环境变量 BAG_DATA_API_URL 获取)
|
|
113
|
+
"""
|
|
114
|
+
# 延迟验证:只有在实际使用时才验证环境变量
|
|
115
|
+
self._app_key = app_key
|
|
116
|
+
self._api_url = api_url
|
|
117
|
+
self._initialized = False
|
|
118
|
+
|
|
119
|
+
def _ensure_initialized(self):
|
|
120
|
+
"""确保服务已初始化(延迟加载配置)"""
|
|
121
|
+
if self._initialized:
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
# 从环境变量获取配置
|
|
125
|
+
if self._app_key is None:
|
|
126
|
+
self._app_key = _get_app_key()
|
|
127
|
+
if self._api_url is None:
|
|
128
|
+
self._api_url = _get_api_url()
|
|
129
|
+
|
|
130
|
+
self._initialized = True
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def app_key(self) -> str:
|
|
134
|
+
"""获取 API 密钥"""
|
|
135
|
+
self._ensure_initialized()
|
|
136
|
+
return self._app_key
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def _api_base_url(self) -> str:
|
|
140
|
+
"""获取 API 地址"""
|
|
141
|
+
self._ensure_initialized()
|
|
142
|
+
return self._api_url
|
|
143
|
+
|
|
144
|
+
def _query_rawdata_by_bag_name(self, bag_name: str) -> Optional[dict]:
|
|
145
|
+
"""通过 bag_name 查询原始数据信息"""
|
|
146
|
+
url = f"{self._api_base_url}/api/rawdata/query"
|
|
147
|
+
headers = {
|
|
148
|
+
"Content-Type": "application/json",
|
|
149
|
+
"x-data-appkey": self.app_key,
|
|
150
|
+
}
|
|
151
|
+
payload = {
|
|
152
|
+
"pageNum": 1,
|
|
153
|
+
"pageSize": 10,
|
|
154
|
+
"bagName": bag_name,
|
|
155
|
+
"sortDesc": True,
|
|
156
|
+
"returnQualityField": True,
|
|
157
|
+
"status": 1,
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
try:
|
|
161
|
+
verify = not _skip_ssl_verify()
|
|
162
|
+
response = requests.post(url, json=payload, headers=headers, timeout=30, verify=verify)
|
|
163
|
+
response.raise_for_status()
|
|
164
|
+
result = response.json()
|
|
165
|
+
|
|
166
|
+
data_list = result.get("data", {}).get("dataList", [])
|
|
167
|
+
if data_list:
|
|
168
|
+
return data_list[0]
|
|
169
|
+
return None
|
|
170
|
+
except Exception as e:
|
|
171
|
+
logger.error(f"查询 rawdata 失败: {e}")
|
|
172
|
+
return None
|
|
173
|
+
|
|
174
|
+
def _parse_bag_info(self, bag_oss_url: str, bag_name: str) -> dict:
|
|
175
|
+
"""
|
|
176
|
+
从 bagOssUrl 解析出日期和车辆信息
|
|
177
|
+
"""
|
|
178
|
+
# 从 bag_name 解析:格式为 {车辆}_{日期}-{时间}_{序号},如 2_033_20260123-225631_0
|
|
179
|
+
parts = bag_name.split("_")
|
|
180
|
+
if len(parts) >= 4:
|
|
181
|
+
# 车辆号:前两部分,如 2_033
|
|
182
|
+
car_name = f"{parts[0]}_{parts[1]}"
|
|
183
|
+
# 日期:第三部分的前8位,如 20260123
|
|
184
|
+
date_time_part = parts[2]
|
|
185
|
+
dt = date_time_part.split("-")[0] if "-" in date_time_part else date_time_part[:8]
|
|
186
|
+
return {"dt": dt, "car_name": car_name}
|
|
187
|
+
|
|
188
|
+
# 备用:从 URL 解析
|
|
189
|
+
try:
|
|
190
|
+
# 移除协议前缀
|
|
191
|
+
path = bag_oss_url.replace("s3://", "").replace("oss://", "")
|
|
192
|
+
path_parts = path.split("/")
|
|
193
|
+
# 查找日期格式的部分(8位数字)
|
|
194
|
+
for part in path_parts:
|
|
195
|
+
if len(part) == 8 and part.isdigit():
|
|
196
|
+
dt = part
|
|
197
|
+
# 日期后面通常是车辆号
|
|
198
|
+
idx = path_parts.index(part)
|
|
199
|
+
if idx + 1 < len(path_parts):
|
|
200
|
+
car_name = path_parts[idx + 1]
|
|
201
|
+
return {"dt": dt, "car_name": car_name}
|
|
202
|
+
except Exception as e:
|
|
203
|
+
logger.warning(f"解析 bag_oss_url 失败: {e}")
|
|
204
|
+
|
|
205
|
+
return {}
|
|
206
|
+
|
|
207
|
+
def _build_target_oss_urls(self, bag_name: str, dt: str, car_name: str) -> dict:
|
|
208
|
+
"""
|
|
209
|
+
构建目标 OSS 地址
|
|
210
|
+
|
|
211
|
+
目标路径格式: oss://{bucket}/{prefix}/{日期}/{车辆}/{bag_name}/{json文件}
|
|
212
|
+
"""
|
|
213
|
+
base_path = f"oss://{TARGET_OSS_BUCKET}/{TARGET_OSS_PREFIX}/{dt}/{car_name}/{bag_name}"
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
"camera_forward_wide": f"{base_path}/camera_forward_wide/camera_forward_wide.json",
|
|
217
|
+
"localization": f"{base_path}/localization/localization.json",
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
def get_bag_json(
|
|
221
|
+
self,
|
|
222
|
+
bag_name: str,
|
|
223
|
+
output_dir: Optional[str] = None,
|
|
224
|
+
print_output: bool = False,
|
|
225
|
+
) -> BagJsonResult:
|
|
226
|
+
"""
|
|
227
|
+
根据 bag_name 获取对应的 JSON 文件 OSS 地址
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
bag_name: 数据包名称,如 "2_033_20260123-225631_0"
|
|
231
|
+
output_dir: 输出目录,如果指定则保存 JSON 文件到该目录
|
|
232
|
+
print_output: 是否打印输出
|
|
233
|
+
|
|
234
|
+
Returns:
|
|
235
|
+
BagJsonResult 对象,包含 camera_forward_wide_url 和 localization_url
|
|
236
|
+
"""
|
|
237
|
+
errors = []
|
|
238
|
+
|
|
239
|
+
# 查询 bag 信息
|
|
240
|
+
raw_data = self._query_rawdata_by_bag_name(bag_name)
|
|
241
|
+
if not raw_data:
|
|
242
|
+
errors.append(f"未找到 bag: {bag_name}")
|
|
243
|
+
return BagJsonResult(bag_name=bag_name, errors=errors)
|
|
244
|
+
|
|
245
|
+
bag_oss_url = raw_data.get("bagOssUrl") or raw_data.get("bagS3Path")
|
|
246
|
+
|
|
247
|
+
# 解析日期和车辆信息
|
|
248
|
+
bag_info = self._parse_bag_info(bag_oss_url or "", bag_name)
|
|
249
|
+
dt = bag_info.get("dt") or raw_data.get("dt")
|
|
250
|
+
car_name = bag_info.get("car_name") or raw_data.get("carName")
|
|
251
|
+
|
|
252
|
+
if not dt or not car_name:
|
|
253
|
+
errors.append(f"无法解析 bag 信息: dt={dt}, car_name={car_name}")
|
|
254
|
+
return BagJsonResult(bag_name=bag_name, errors=errors)
|
|
255
|
+
|
|
256
|
+
# 构建目标 OSS 地址
|
|
257
|
+
oss_urls = self._build_target_oss_urls(bag_name, dt, car_name)
|
|
258
|
+
|
|
259
|
+
result = BagJsonResult(
|
|
260
|
+
bag_name=bag_name,
|
|
261
|
+
camera_forward_wide_url=oss_urls["camera_forward_wide"],
|
|
262
|
+
localization_url=oss_urls["localization"],
|
|
263
|
+
errors=errors if errors else None,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
# 打印输出
|
|
267
|
+
if print_output:
|
|
268
|
+
print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2))
|
|
269
|
+
|
|
270
|
+
# 保存到文件
|
|
271
|
+
if output_dir:
|
|
272
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
273
|
+
output_file = os.path.join(output_dir, f"{bag_name}.json")
|
|
274
|
+
with open(output_file, "w", encoding="utf-8") as f:
|
|
275
|
+
json.dump(result.to_dict(), f, ensure_ascii=False, indent=2)
|
|
276
|
+
|
|
277
|
+
return result
|
|
278
|
+
|
|
279
|
+
def get_bag_json_batch(
|
|
280
|
+
self,
|
|
281
|
+
bag_names: list[str],
|
|
282
|
+
output_dir: Optional[str] = None,
|
|
283
|
+
print_output: bool = False,
|
|
284
|
+
) -> dict[str, BagJsonResult]:
|
|
285
|
+
"""
|
|
286
|
+
批量获取多个 bag 的 JSON 文件 OSS 地址
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
bag_names: 数据包名称列表
|
|
290
|
+
output_dir: 输出目录
|
|
291
|
+
print_output: 是否打印输出
|
|
292
|
+
|
|
293
|
+
Returns:
|
|
294
|
+
字典,key 为 bag_name,value 为 BagJsonResult
|
|
295
|
+
"""
|
|
296
|
+
results = {}
|
|
297
|
+
for bag_name in bag_names:
|
|
298
|
+
results[bag_name] = self.get_bag_json(
|
|
299
|
+
bag_name,
|
|
300
|
+
output_dir=output_dir,
|
|
301
|
+
print_output=print_output,
|
|
302
|
+
)
|
|
303
|
+
return results
|
|
304
|
+
|
|
305
|
+
def get_bag_info(self, bag_name: str) -> Optional[dict]:
|
|
306
|
+
"""获取 bag 的基本信息"""
|
|
307
|
+
raw_data = self._query_rawdata_by_bag_name(bag_name)
|
|
308
|
+
if not raw_data:
|
|
309
|
+
return None
|
|
310
|
+
|
|
311
|
+
bag_oss_url = raw_data.get("bagOssUrl") or raw_data.get("bagS3Path")
|
|
312
|
+
bag_info = self._parse_bag_info(bag_oss_url or "", bag_name)
|
|
313
|
+
dt = bag_info.get("dt") or raw_data.get("dt")
|
|
314
|
+
car_name = bag_info.get("car_name") or raw_data.get("carName")
|
|
315
|
+
|
|
316
|
+
oss_urls = self._build_target_oss_urls(bag_name, dt, car_name) if dt and car_name else {}
|
|
317
|
+
|
|
318
|
+
return {
|
|
319
|
+
"bag_name": bag_name,
|
|
320
|
+
"bag_key": raw_data.get("bagKey"),
|
|
321
|
+
"car_name": raw_data.get("carName"),
|
|
322
|
+
"city_name": raw_data.get("cityName"),
|
|
323
|
+
"dt": raw_data.get("dt"),
|
|
324
|
+
"bag_oss_url": bag_oss_url,
|
|
325
|
+
"target_oss_urls": oss_urls,
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
def query_and_export(
|
|
329
|
+
self,
|
|
330
|
+
output_dir: Optional[str] = None,
|
|
331
|
+
page_num: int = 1,
|
|
332
|
+
page_size: int = 20,
|
|
333
|
+
car_nos: Optional[list[str]] = None,
|
|
334
|
+
data_version: Optional[str] = None,
|
|
335
|
+
begin_time_from: Optional[int] = None,
|
|
336
|
+
begin_time_to: Optional[int] = None,
|
|
337
|
+
**kwargs,
|
|
338
|
+
) -> dict:
|
|
339
|
+
"""
|
|
340
|
+
查询 Bag 数据并获取 OSS 地址
|
|
341
|
+
|
|
342
|
+
Args:
|
|
343
|
+
output_dir: 输出目录,如果指定则保存 JSON 文件
|
|
344
|
+
page_num: 页码
|
|
345
|
+
page_size: 每页数量
|
|
346
|
+
car_nos: 车辆号列表,如 ["1_023", "1_036"]
|
|
347
|
+
data_version: 数据版本,如 "V1.4.1"
|
|
348
|
+
begin_time_from: 开始时间(毫秒时间戳)
|
|
349
|
+
begin_time_to: 结束时间(毫秒时间戳)
|
|
350
|
+
**kwargs: 其他查询参数
|
|
351
|
+
|
|
352
|
+
Returns:
|
|
353
|
+
字典,包含 query_result(查询结果)和 export_results(导出结果)
|
|
354
|
+
|
|
355
|
+
Example:
|
|
356
|
+
service = BagDataService()
|
|
357
|
+
|
|
358
|
+
result = service.query_and_export(
|
|
359
|
+
output_dir="./output",
|
|
360
|
+
car_nos=["2_033"],
|
|
361
|
+
page_size=10,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
for bag_name, data in result['export_results'].items():
|
|
365
|
+
print(f"{bag_name}:")
|
|
366
|
+
print(f" 前广角: {data.camera_forward_wide_url}")
|
|
367
|
+
print(f" 定位: {data.localization_url}")
|
|
368
|
+
"""
|
|
369
|
+
# 构建查询参数
|
|
370
|
+
query_params = {
|
|
371
|
+
"pageNum": page_num,
|
|
372
|
+
"pageSize": page_size,
|
|
373
|
+
"sortDesc": True,
|
|
374
|
+
"returnQualityField": True,
|
|
375
|
+
"status": 1,
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
if car_nos:
|
|
379
|
+
query_params["carNos"] = car_nos
|
|
380
|
+
if data_version:
|
|
381
|
+
query_params["dataVersion"] = data_version
|
|
382
|
+
if begin_time_from:
|
|
383
|
+
query_params["beginTimeFrom"] = begin_time_from
|
|
384
|
+
if begin_time_to:
|
|
385
|
+
query_params["beginTimeTo"] = begin_time_to
|
|
386
|
+
query_params.update(kwargs)
|
|
387
|
+
|
|
388
|
+
# 执行查询
|
|
389
|
+
url = f"{self._api_base_url}/api/rawdata/query"
|
|
390
|
+
headers = {
|
|
391
|
+
"Content-Type": "application/json",
|
|
392
|
+
"x-data-appkey": self.app_key,
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
verify = not _skip_ssl_verify()
|
|
396
|
+
response = requests.post(url, json=query_params, headers=headers, timeout=30, verify=verify)
|
|
397
|
+
response.raise_for_status()
|
|
398
|
+
api_result = response.json()
|
|
399
|
+
|
|
400
|
+
data_list = api_result.get("data", {}).get("dataList", [])
|
|
401
|
+
total_count = api_result.get("data", {}).get("totalCount", -1)
|
|
402
|
+
|
|
403
|
+
# 获取 OSS 地址
|
|
404
|
+
export_results = {}
|
|
405
|
+
|
|
406
|
+
if output_dir:
|
|
407
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
408
|
+
|
|
409
|
+
for item in data_list:
|
|
410
|
+
bag_name = item.get("bagName")
|
|
411
|
+
if not bag_name:
|
|
412
|
+
continue
|
|
413
|
+
|
|
414
|
+
result = self.get_bag_json(bag_name, output_dir=output_dir, print_output=False)
|
|
415
|
+
export_results[bag_name] = result
|
|
416
|
+
|
|
417
|
+
return {
|
|
418
|
+
"query_result": data_list,
|
|
419
|
+
"total_count": total_count,
|
|
420
|
+
"export_results": export_results,
|
|
421
|
+
"output_dir": output_dir,
|
|
422
|
+
}
|
|
@@ -122,6 +122,100 @@ def find_service_classes(
|
|
|
122
122
|
return service_classes
|
|
123
123
|
|
|
124
124
|
|
|
125
|
+
def _load_class_by_path(class_path: str, search_path: str = ".") -> Type[BaseService]:
|
|
126
|
+
"""
|
|
127
|
+
根据相对路径加载服务类
|
|
128
|
+
|
|
129
|
+
支持的格式:
|
|
130
|
+
- "ClassName" - 仅类名,从所有发现的服务中查找
|
|
131
|
+
- "module.ClassName" - 模块名.类名
|
|
132
|
+
- "path/to/module.ClassName" - 相对路径/模块名.类名
|
|
133
|
+
- "path.to.module.ClassName" - 点分隔的完整路径
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
class_path: 类的路径,支持多种格式
|
|
137
|
+
search_path: 搜索的根路径
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
服务类
|
|
141
|
+
|
|
142
|
+
Raises:
|
|
143
|
+
ValueError: 如果找不到指定的类
|
|
144
|
+
"""
|
|
145
|
+
import sys
|
|
146
|
+
import os
|
|
147
|
+
|
|
148
|
+
# 确保搜索路径在 Python 路径中
|
|
149
|
+
search_path_obj = Path(search_path).resolve()
|
|
150
|
+
search_path_str = str(search_path_obj)
|
|
151
|
+
|
|
152
|
+
current_dir = os.getcwd()
|
|
153
|
+
if current_dir not in sys.path:
|
|
154
|
+
sys.path.insert(0, current_dir)
|
|
155
|
+
if search_path_str not in sys.path:
|
|
156
|
+
sys.path.insert(0, search_path_str)
|
|
157
|
+
if '.' not in sys.path:
|
|
158
|
+
sys.path.insert(0, '.')
|
|
159
|
+
|
|
160
|
+
# 解析 class_path
|
|
161
|
+
# 如果不包含 '.' 则认为只是类名
|
|
162
|
+
if '.' not in class_path and '/' not in class_path:
|
|
163
|
+
# 仅类名,返回 None 表示需要从发现的服务中查找
|
|
164
|
+
return None
|
|
165
|
+
|
|
166
|
+
# 将路径分隔符统一转换为点分隔
|
|
167
|
+
# 例如: "path/to/module.ClassName" -> "path.to.module.ClassName"
|
|
168
|
+
normalized_path = class_path.replace('/', '.').replace('\\', '.')
|
|
169
|
+
|
|
170
|
+
# 去掉 .py 后缀(如果有)
|
|
171
|
+
if '.py.' in normalized_path:
|
|
172
|
+
normalized_path = normalized_path.replace('.py.', '.')
|
|
173
|
+
elif normalized_path.endswith('.py'):
|
|
174
|
+
normalized_path = normalized_path[:-3]
|
|
175
|
+
|
|
176
|
+
# 分离模块路径和类名
|
|
177
|
+
# 最后一个点之后的是类名
|
|
178
|
+
parts = normalized_path.rsplit('.', 1)
|
|
179
|
+
if len(parts) != 2:
|
|
180
|
+
raise ValueError(
|
|
181
|
+
f"Invalid class path format: '{class_path}'. "
|
|
182
|
+
f"Expected format: 'module.ClassName' or 'path/to/module.ClassName'"
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
module_path, class_name = parts
|
|
186
|
+
|
|
187
|
+
try:
|
|
188
|
+
# 动态导入模块
|
|
189
|
+
module = importlib.import_module(module_path)
|
|
190
|
+
|
|
191
|
+
# 获取类
|
|
192
|
+
if not hasattr(module, class_name):
|
|
193
|
+
raise ValueError(
|
|
194
|
+
f"Class '{class_name}' not found in module '{module_path}'"
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
cls = getattr(module, class_name)
|
|
198
|
+
|
|
199
|
+
# 验证是否是 BaseService 的子类
|
|
200
|
+
if not (inspect.isclass(cls) and issubclass(cls, BaseService)):
|
|
201
|
+
raise ValueError(
|
|
202
|
+
f"Class '{class_name}' is not a subclass of BaseService"
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
if inspect.isabstract(cls):
|
|
206
|
+
raise ValueError(
|
|
207
|
+
f"Class '{class_name}' is abstract and cannot be instantiated"
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
return cls
|
|
211
|
+
|
|
212
|
+
except ImportError as e:
|
|
213
|
+
raise ValueError(
|
|
214
|
+
f"Failed to import module '{module_path}': {e}\n"
|
|
215
|
+
f"Make sure the module path is correct and all dependencies are installed."
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
|
|
125
219
|
def get_single_service_class(
|
|
126
220
|
search_path: str = ".",
|
|
127
221
|
class_name: Optional[str] = None
|
|
@@ -131,7 +225,12 @@ def get_single_service_class(
|
|
|
131
225
|
|
|
132
226
|
Args:
|
|
133
227
|
search_path: 搜索路径
|
|
134
|
-
class_name:
|
|
228
|
+
class_name: 指定的类名或完整路径(可选)
|
|
229
|
+
支持的格式:
|
|
230
|
+
- "ClassName" - 仅类名,从所有发现的服务中查找
|
|
231
|
+
- "module.ClassName" - 模块名.类名
|
|
232
|
+
- "path/to/module.ClassName" - 相对路径/模块名.类名
|
|
233
|
+
- "path.to.module.ClassName" - 点分隔的完整路径
|
|
135
234
|
|
|
136
235
|
Returns:
|
|
137
236
|
服务类
|
|
@@ -139,6 +238,13 @@ def get_single_service_class(
|
|
|
139
238
|
Raises:
|
|
140
239
|
ValueError: 如果找到0个或多个服务类
|
|
141
240
|
"""
|
|
241
|
+
# 如果指定了包含路径的类名,尝试直接加载
|
|
242
|
+
if class_name:
|
|
243
|
+
direct_class = _load_class_by_path(class_name, search_path)
|
|
244
|
+
if direct_class is not None:
|
|
245
|
+
return direct_class
|
|
246
|
+
# 如果返回 None,说明只是类名,继续使用发现机制
|
|
247
|
+
|
|
142
248
|
service_classes = find_service_classes(search_path)
|
|
143
249
|
|
|
144
250
|
if not service_classes:
|
|
@@ -166,11 +272,15 @@ def get_single_service_class(
|
|
|
166
272
|
for f in possible_files:
|
|
167
273
|
error_msg += f" - {f}\n"
|
|
168
274
|
error_msg += f"\nTry using --class-name to specify the service class name."
|
|
275
|
+
error_msg += f"\nSupported formats:"
|
|
276
|
+
error_msg += f"\n - --class-name ClassName"
|
|
277
|
+
error_msg += f"\n - --class-name module.ClassName"
|
|
278
|
+
error_msg += f"\n - --class-name path/to/module.ClassName"
|
|
169
279
|
|
|
170
280
|
raise ValueError(error_msg)
|
|
171
281
|
|
|
172
282
|
if class_name:
|
|
173
|
-
#
|
|
283
|
+
# 查找指定类名的服务(此时 class_name 只是类名,不包含路径)
|
|
174
284
|
for module_path, cls in service_classes:
|
|
175
285
|
if cls.__name__ == class_name:
|
|
176
286
|
return cls
|
|
@@ -181,7 +291,11 @@ def get_single_service_class(
|
|
|
181
291
|
class_list = [f"{module}.{cls.__name__}" for module, cls in service_classes]
|
|
182
292
|
raise ValueError(
|
|
183
293
|
f"Multiple service classes found: {class_list}. "
|
|
184
|
-
f"Please specify which one to use with --class-name
|
|
294
|
+
f"Please specify which one to use with --class-name.\n"
|
|
295
|
+
f"Supported formats:\n"
|
|
296
|
+
f" - --class-name ClassName\n"
|
|
297
|
+
f" - --class-name module.ClassName\n"
|
|
298
|
+
f" - --class-name path/to/module.ClassName"
|
|
185
299
|
)
|
|
186
300
|
|
|
187
301
|
return service_classes[0][1]
|
{hello_datap_component_base-0.2.3.dist-info → hello_datap_component_base-0.2.5.dist-info}/METADATA
RENAMED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hello-datap-component-base
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: A unified server management framework for data processing component
|
|
5
|
-
Author
|
|
5
|
+
Author: Data Processing Team
|
|
6
6
|
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://gitlab.hellorobotaxi.top/hdata/hello-datap-component-base
|
|
8
7
|
Keywords: data,hello,management,microservice
|
|
9
8
|
Classifier: Development Status :: 4 - Beta
|
|
10
9
|
Classifier: Intended Audience :: Developers
|
|
@@ -24,6 +23,7 @@ Requires-Dist: python-json-logger>=2.0.0
|
|
|
24
23
|
Requires-Dist: pyyaml>=6.0.0
|
|
25
24
|
Requires-Dist: aliyun-mns>=1.1.5
|
|
26
25
|
Requires-Dist: oss2>=2.18.0
|
|
26
|
+
Requires-Dist: requests>=2.28.0
|
|
27
27
|
Provides-Extra: dev
|
|
28
28
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
29
|
Requires-Dist: black>=23.0.0; extra == "dev"
|
|
@@ -380,7 +380,11 @@ hello-datap-component-base/
|
|
|
380
380
|
│ ├── cli.py # 命令行工具
|
|
381
381
|
│ ├── discover.py # 服务发现
|
|
382
382
|
│ ├── logger.py # 日志管理
|
|
383
|
-
│
|
|
383
|
+
│ ├── mns_client.py # MNS 队列客户端
|
|
384
|
+
│ ├── oss_client.py # OSS 客户端
|
|
385
|
+
│ └── data/ # 数据服务模块
|
|
386
|
+
│ ├── __init__.py
|
|
387
|
+
│ └── bag_data_service.py # Bag 数据服务
|
|
384
388
|
├── example_service.py # 使用示例
|
|
385
389
|
├── example_config.json # 示例配置
|
|
386
390
|
├── README.md # 本文档
|
|
@@ -641,6 +645,27 @@ A: 在配置文件的 `runtime_env.env_vars` 中设置 OSS 相关环境变量:
|
|
|
641
645
|
|
|
642
646
|
**向下兼容**:如果未配置 OSS 环境变量,保持现状(不上传),不影响主流程。
|
|
643
647
|
|
|
648
|
+
**Q: 如何使用 BagDataService 获取 Bag 数据?**
|
|
649
|
+
|
|
650
|
+
A: BagDataService 用于根据 bag_name 获取对应的 JSON 文件 OSS 地址。首先配置环境变量,然后使用服务:
|
|
651
|
+
|
|
652
|
+
```bash
|
|
653
|
+
# 设置环境变量
|
|
654
|
+
export BAG_DATA_APP_KEY=your_app_key
|
|
655
|
+
export BAG_DATA_API_URL=https://your-api-url.com
|
|
656
|
+
```
|
|
657
|
+
|
|
658
|
+
```python
|
|
659
|
+
from hello_datap_component_base import BagDataService
|
|
660
|
+
|
|
661
|
+
service = BagDataService()
|
|
662
|
+
result = service.get_bag_json("2_033_20260123-225631_0")
|
|
663
|
+
print(result.camera_forward_wide_url) # OSS 地址
|
|
664
|
+
print(result.localization_url) # OSS 地址
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
详细用法请参考下方 "BagDataService 数据服务" 章节。
|
|
668
|
+
|
|
644
669
|
**Q: 返回结果格式是什么?**
|
|
645
670
|
|
|
646
671
|
A: `handle_request` 方法会自动封装返回结果:
|
|
@@ -648,11 +673,83 @@ A: `handle_request` 方法会自动封装返回结果:
|
|
|
648
673
|
- 异常情况:`code=-1, message=错误消息, data.out_put=null`
|
|
649
674
|
- 所有结果都包含 `work_flow_id`、`work_flow_instance_id`、`task_id`
|
|
650
675
|
|
|
676
|
+
## BagDataService 数据服务
|
|
677
|
+
|
|
678
|
+
BagDataService 用于根据 bag_name 获取对应的 JSON 文件 OSS 地址。
|
|
679
|
+
|
|
680
|
+
### 环境变量配置
|
|
681
|
+
|
|
682
|
+
使用 BagDataService 前,必须设置以下环境变量:
|
|
683
|
+
|
|
684
|
+
| 环境变量 | 说明 | 必需 |
|
|
685
|
+
|---------|------|------|
|
|
686
|
+
| `BAG_DATA_APP_KEY` | API 密钥 | 是 |
|
|
687
|
+
| `BAG_DATA_API_URL` | API 服务地址 | 是 |
|
|
688
|
+
| `SKIP_SSL_VERIFY` | 是否跳过 SSL 验证(可选,默认 false) | 否 |
|
|
689
|
+
|
|
690
|
+
```bash
|
|
691
|
+
# 设置环境变量
|
|
692
|
+
export BAG_DATA_APP_KEY=your_app_key
|
|
693
|
+
export BAG_DATA_API_URL=https://your-api-url.com
|
|
694
|
+
|
|
695
|
+
# 可选:跳过 SSL 验证(仅在内部服务使用)
|
|
696
|
+
export SKIP_SSL_VERIFY=true
|
|
697
|
+
```
|
|
698
|
+
|
|
699
|
+
### 使用示例
|
|
700
|
+
|
|
701
|
+
```python
|
|
702
|
+
from hello_datap_component_base import BagDataService
|
|
703
|
+
|
|
704
|
+
# 创建服务实例(配置从环境变量读取)
|
|
705
|
+
service = BagDataService()
|
|
706
|
+
|
|
707
|
+
# 获取单个 bag 的 OSS 地址
|
|
708
|
+
result = service.get_bag_json("2_033_20260123-225631_0")
|
|
709
|
+
print(result.camera_forward_wide_url) # OSS 地址
|
|
710
|
+
print(result.localization_url) # OSS 地址
|
|
711
|
+
|
|
712
|
+
# 批量获取
|
|
713
|
+
results = service.get_bag_json_batch(["bag1", "bag2"])
|
|
714
|
+
for bag_name, data in results.items():
|
|
715
|
+
print(f"{bag_name}: {data.camera_forward_wide_url}")
|
|
716
|
+
|
|
717
|
+
# 查询并导出
|
|
718
|
+
result = service.query_and_export(
|
|
719
|
+
output_dir="./output",
|
|
720
|
+
car_nos=["2_033"],
|
|
721
|
+
page_size=10,
|
|
722
|
+
)
|
|
723
|
+
```
|
|
724
|
+
|
|
725
|
+
### 返回结果
|
|
726
|
+
|
|
727
|
+
`BagJsonResult` 对象包含以下字段:
|
|
728
|
+
|
|
729
|
+
- `bag_name`: 数据包名称
|
|
730
|
+
- `camera_forward_wide_url`: 前广角相机 JSON 文件 OSS 地址
|
|
731
|
+
- `localization_url`: 定位 JSON 文件 OSS 地址
|
|
732
|
+
- `errors`: 错误信息列表(如有)
|
|
733
|
+
|
|
734
|
+
### 在配置文件中使用
|
|
735
|
+
|
|
736
|
+
如果在服务运行时需要使用 BagDataService,可以在配置文件中设置环境变量:
|
|
737
|
+
|
|
738
|
+
```json
|
|
739
|
+
{
|
|
740
|
+
"runtime_env": {
|
|
741
|
+
"env_vars": {
|
|
742
|
+
"BAG_DATA_APP_KEY": "your_app_key",
|
|
743
|
+
"BAG_DATA_API_URL": "https://your-api-url.com"
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
```
|
|
748
|
+
|
|
651
749
|
## 许可证
|
|
652
750
|
|
|
653
751
|
MIT License
|
|
654
752
|
|
|
655
753
|
## 作者
|
|
656
754
|
|
|
657
|
-
|
|
658
|
-
|
|
755
|
+
Data Processing Team
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
hello_datap_component_base/__init__.py,sha256=Biy9-ElF10_cJCQNYRxHHx9NC_zrOzEQ4YjG6-HGxTA,909
|
|
2
|
+
hello_datap_component_base/base.py,sha256=jkjoUx9QQ4IqiwR0WZrb2-ZX9KEKF1_fOF_415qwsxc,6436
|
|
3
|
+
hello_datap_component_base/cli.py,sha256=T_rcmjhZMuMhi7av-MwE3d06XwX_TGLsYs-WzKTLQk4,8758
|
|
4
|
+
hello_datap_component_base/config.py,sha256=XV4OY0iCEjVf0PNxLRdWLgOB5pPB0OvASdkysZXukms,6992
|
|
5
|
+
hello_datap_component_base/discover.py,sha256=iSxtfBTxhEbN8m9eDQ1OCEys0Nr-mzDK756mBWFtGoI,11644
|
|
6
|
+
hello_datap_component_base/logger.py,sha256=JIvy_gctDf0Vpe_itSQCwf-ZhVigMdDFwpwbmMlcNNE,10606
|
|
7
|
+
hello_datap_component_base/mns_client.py,sha256=uaWb4P99iRk8vpREqEDqjD9HqaKwlhOYEtHfaW_Y2Pg,12521
|
|
8
|
+
hello_datap_component_base/oss_client.py,sha256=C4Dajeey3JBKh_EgGJzzNMMdpzkm85Z6tpO1kC1wU_s,5057
|
|
9
|
+
hello_datap_component_base/runner.py,sha256=korZY_Qoa-ZzwIFsWK4zaetLF17BE9oT-IRDueoaSbs,11576
|
|
10
|
+
hello_datap_component_base/data/__init__.py,sha256=3A6giCgxL-s0zWluDzPvh3HiAzriPQUzHgGPjayowsk,184
|
|
11
|
+
hello_datap_component_base/data/bag_data_service.py,sha256=h8oU4okqDd6mkovbIGkW6lRsEswyAGHCz9DsFQQX8s8,14503
|
|
12
|
+
hello_datap_component_base-0.2.5.dist-info/METADATA,sha256=g8gkv5aAXAueEaQRiHy_E_qlSskoVoJlu7P7yozaZeM,22438
|
|
13
|
+
hello_datap_component_base-0.2.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
14
|
+
hello_datap_component_base-0.2.5.dist-info/entry_points.txt,sha256=Q2YteaAVN0UW9MEBfPZ3EY6FI6dRaoCmQZpcvAzmSVQ,74
|
|
15
|
+
hello_datap_component_base-0.2.5.dist-info/top_level.txt,sha256=V4aKCkt0lrJbO4RBPymJ6dz5mXo6vjYy02kusdWKGqM,27
|
|
16
|
+
hello_datap_component_base-0.2.5.dist-info/RECORD,,
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
hello_datap_component_base/__init__.py,sha256=5bGlAUivYRN9_H91I9tYoUrokl6YJJgJGtlM4PkLU_A,811
|
|
2
|
-
hello_datap_component_base/base.py,sha256=jkjoUx9QQ4IqiwR0WZrb2-ZX9KEKF1_fOF_415qwsxc,6436
|
|
3
|
-
hello_datap_component_base/cli.py,sha256=chRlcOTaqHJFdKNPe0OTFYvw_M4c_1_kHBlQ82BwB4g,8161
|
|
4
|
-
hello_datap_component_base/config.py,sha256=XV4OY0iCEjVf0PNxLRdWLgOB5pPB0OvASdkysZXukms,6992
|
|
5
|
-
hello_datap_component_base/discover.py,sha256=70sFO9iVnpsjo_eTViQsStI-n0N_eJNvDRvLvm_dqZQ,7459
|
|
6
|
-
hello_datap_component_base/logger.py,sha256=JIvy_gctDf0Vpe_itSQCwf-ZhVigMdDFwpwbmMlcNNE,10606
|
|
7
|
-
hello_datap_component_base/mns_client.py,sha256=uaWb4P99iRk8vpREqEDqjD9HqaKwlhOYEtHfaW_Y2Pg,12521
|
|
8
|
-
hello_datap_component_base/oss_client.py,sha256=C4Dajeey3JBKh_EgGJzzNMMdpzkm85Z6tpO1kC1wU_s,5057
|
|
9
|
-
hello_datap_component_base/runner.py,sha256=korZY_Qoa-ZzwIFsWK4zaetLF17BE9oT-IRDueoaSbs,11576
|
|
10
|
-
hello_datap_component_base-0.2.3.dist-info/METADATA,sha256=fHLMMFXgwraItePBCFDAHvCUf4bnW8JHE85yAyeiobQ,19865
|
|
11
|
-
hello_datap_component_base-0.2.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
12
|
-
hello_datap_component_base-0.2.3.dist-info/entry_points.txt,sha256=Q2YteaAVN0UW9MEBfPZ3EY6FI6dRaoCmQZpcvAzmSVQ,74
|
|
13
|
-
hello_datap_component_base-0.2.3.dist-info/top_level.txt,sha256=V4aKCkt0lrJbO4RBPymJ6dz5mXo6vjYy02kusdWKGqM,27
|
|
14
|
-
hello_datap_component_base-0.2.3.dist-info/RECORD,,
|
{hello_datap_component_base-0.2.3.dist-info → hello_datap_component_base-0.2.5.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|