fraclab-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1601 -0
- fraclab_sdk/__init__.py +34 -0
- fraclab_sdk/algorithm/__init__.py +13 -0
- fraclab_sdk/algorithm/export.py +1 -0
- fraclab_sdk/algorithm/library.py +378 -0
- fraclab_sdk/cli.py +381 -0
- fraclab_sdk/config.py +54 -0
- fraclab_sdk/devkit/__init__.py +25 -0
- fraclab_sdk/devkit/compile.py +342 -0
- fraclab_sdk/devkit/export.py +354 -0
- fraclab_sdk/devkit/validate.py +1043 -0
- fraclab_sdk/errors.py +124 -0
- fraclab_sdk/materialize/__init__.py +8 -0
- fraclab_sdk/materialize/fsops.py +125 -0
- fraclab_sdk/materialize/hash.py +28 -0
- fraclab_sdk/materialize/materializer.py +241 -0
- fraclab_sdk/models/__init__.py +52 -0
- fraclab_sdk/models/bundle_manifest.py +51 -0
- fraclab_sdk/models/dataspec.py +65 -0
- fraclab_sdk/models/drs.py +47 -0
- fraclab_sdk/models/output_contract.py +111 -0
- fraclab_sdk/models/run_output_manifest.py +119 -0
- fraclab_sdk/results/__init__.py +25 -0
- fraclab_sdk/results/preview.py +150 -0
- fraclab_sdk/results/reader.py +329 -0
- fraclab_sdk/run/__init__.py +10 -0
- fraclab_sdk/run/logs.py +42 -0
- fraclab_sdk/run/manager.py +403 -0
- fraclab_sdk/run/subprocess_runner.py +153 -0
- fraclab_sdk/runtime/__init__.py +11 -0
- fraclab_sdk/runtime/artifacts.py +303 -0
- fraclab_sdk/runtime/data_client.py +123 -0
- fraclab_sdk/runtime/runner_main.py +286 -0
- fraclab_sdk/runtime/snapshot_provider.py +1 -0
- fraclab_sdk/selection/__init__.py +11 -0
- fraclab_sdk/selection/model.py +247 -0
- fraclab_sdk/selection/validate.py +54 -0
- fraclab_sdk/snapshot/__init__.py +12 -0
- fraclab_sdk/snapshot/index.py +94 -0
- fraclab_sdk/snapshot/library.py +205 -0
- fraclab_sdk/snapshot/loader.py +217 -0
- fraclab_sdk/specs/manifest.py +89 -0
- fraclab_sdk/utils/io.py +32 -0
- fraclab_sdk-0.1.0.dist-info/METADATA +1622 -0
- fraclab_sdk-0.1.0.dist-info/RECORD +47 -0
- fraclab_sdk-0.1.0.dist-info/WHEEL +4 -0
- fraclab_sdk-0.1.0.dist-info/entry_points.txt +4 -0
|
@@ -0,0 +1,1622 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fraclab-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: SDK for managing snapshots, algorithms, and run execution
|
|
5
|
+
Requires-Python: >=3.11,<4.0
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Requires-Dist: fastapi (>=0.115.0,<0.116.0)
|
|
12
|
+
Requires-Dist: matplotlib (>=3.9.2,<4.0.0)
|
|
13
|
+
Requires-Dist: numpy (>=2.1.1,<3.0.0)
|
|
14
|
+
Requires-Dist: pandas (>=2.2.3,<3.0.0)
|
|
15
|
+
Requires-Dist: pydantic (>=2.10.0,<3.0.0)
|
|
16
|
+
Requires-Dist: rich (>=13.9.0,<14.0.0)
|
|
17
|
+
Requires-Dist: scipy (>=1.13.1,<2.0.0)
|
|
18
|
+
Requires-Dist: typer (>=0.15.0,<0.16.0)
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# Fraclab SDK Reference
|
|
22
|
+
|
|
23
|
+
> 版本: 0.1.0
|
|
24
|
+
> Python: >=3.11
|
|
25
|
+
|
|
26
|
+
Fraclab SDK 是一个算法开发与执行框架,帮助算法开发者快速构建、测试和部署数据处理算法。
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## 目录
|
|
31
|
+
|
|
32
|
+
1. [安装](#安装)
|
|
33
|
+
2. [快速开始:编写你的第一个算法](#快速开始编写你的第一个算法)
|
|
34
|
+
3. [Bundle 与 Snapshot](#bundle-与-snapshot)
|
|
35
|
+
4. [算法开发详解](#算法开发详解)
|
|
36
|
+
5. [CLI 命令行工具](#cli-命令行工具)
|
|
37
|
+
6. [SDK 内部模块](#sdk-内部模块)
|
|
38
|
+
7. [数据模型](#数据模型)
|
|
39
|
+
8. [错误处理](#错误处理)
|
|
40
|
+
9. [附录 A: Bundle 结构详解](#附录-a-bundle-结构详解)
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## 安装
|
|
45
|
+
|
|
46
|
+
使用 wheel 文件安装:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install fraclab_sdk-0.1.0-py3-none-any.whl
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 快速开始:编写你的第一个算法
|
|
55
|
+
|
|
56
|
+
### 1. 导入运行时组件
|
|
57
|
+
|
|
58
|
+
算法开发者主要使用 `fraclab_sdk.runtime` 模块中的两个核心类:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from fraclab_sdk.runtime import DataClient, ArtifactWriter
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
- **DataClient**: 读取输入数据
|
|
65
|
+
- **ArtifactWriter**: 写入输出结果
|
|
66
|
+
|
|
67
|
+
### 2. 编写算法入口
|
|
68
|
+
|
|
69
|
+
创建 `main.py` 作为算法入口文件:
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
# main.py
|
|
73
|
+
from fraclab_sdk.runtime import DataClient, ArtifactWriter
|
|
74
|
+
from pathlib import Path
|
|
75
|
+
|
|
76
|
+
def run(ctx):
|
|
77
|
+
"""算法入口函数。
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
ctx: RunContext,包含:
|
|
81
|
+
- ctx.data_client: DataClient 实例
|
|
82
|
+
- ctx.params: dict,用户参数
|
|
83
|
+
- ctx.artifacts: ArtifactWriter 实例
|
|
84
|
+
- ctx.logger: Logger 实例
|
|
85
|
+
- ctx.run_context: dict,运行上下文
|
|
86
|
+
"""
|
|
87
|
+
dc = ctx.data_client
|
|
88
|
+
aw = ctx.artifacts
|
|
89
|
+
params = ctx.params
|
|
90
|
+
logger = ctx.logger
|
|
91
|
+
|
|
92
|
+
# 获取参数
|
|
93
|
+
threshold = params.get("threshold", 0.5)
|
|
94
|
+
logger.info(f"开始处理,阈值: {threshold}")
|
|
95
|
+
|
|
96
|
+
# 读取输入数据
|
|
97
|
+
for dataset_key in dc.get_dataset_keys():
|
|
98
|
+
count = dc.get_item_count(dataset_key)
|
|
99
|
+
logger.info(f"数据集 {dataset_key} 包含 {count} 个项目")
|
|
100
|
+
|
|
101
|
+
for i in range(count):
|
|
102
|
+
# 读取 NDJSON 对象
|
|
103
|
+
obj = dc.read_object(dataset_key, i)
|
|
104
|
+
|
|
105
|
+
# 处理数据...
|
|
106
|
+
result = process(obj, threshold)
|
|
107
|
+
|
|
108
|
+
# 写入结果
|
|
109
|
+
aw.write_scalar(f"{dataset_key}_result_{i}", result)
|
|
110
|
+
|
|
111
|
+
# 写入汇总结果
|
|
112
|
+
aw.write_json("summary", {"status": "completed", "threshold": threshold})
|
|
113
|
+
|
|
114
|
+
logger.info("算法执行完成")
|
|
115
|
+
|
|
116
|
+
def process(data, threshold):
|
|
117
|
+
"""你的数据处理逻辑"""
|
|
118
|
+
return data.get("value", 0) > threshold
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### 3. 定义输入参数规格 (InputSpec)
|
|
122
|
+
|
|
123
|
+
创建 `schema/inputspec.py` 定义算法接受的参数:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
# schema/inputspec.py
|
|
127
|
+
from pydantic import BaseModel, Field
|
|
128
|
+
|
|
129
|
+
class InputParams(BaseModel):
|
|
130
|
+
threshold: float = Field(
|
|
131
|
+
default=0.5,
|
|
132
|
+
ge=0.0,
|
|
133
|
+
le=1.0,
|
|
134
|
+
description="检测阈值"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
debug: bool = Field(
|
|
138
|
+
default=False,
|
|
139
|
+
description="启用调试模式"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# 必须导出 INPUT_SPEC
|
|
143
|
+
INPUT_SPEC = InputParams
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
#### ctx.params 的类型与访问方式
|
|
147
|
+
|
|
148
|
+
**`ctx.params` 是 `dict[str, Any]` 类型**,键名来自 `params.json` (JSON 原始键名)。
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
# main.py - 算法代码
|
|
152
|
+
def run(ctx):
|
|
153
|
+
# ctx.params 是 dict,使用 dict 访问方式
|
|
154
|
+
threshold = ctx.params.get("threshold", 0.5)
|
|
155
|
+
debug = ctx.params.get("debug", False)
|
|
156
|
+
|
|
157
|
+
# 嵌套对象同样是 dict
|
|
158
|
+
filters = ctx.params.get("filters", {})
|
|
159
|
+
min_depth = filters.get("minDepth", 0)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
#### InputSpec 与 JSON 命名规则
|
|
163
|
+
|
|
164
|
+
| 层级 | 命名风格 | 示例 | 说明 |
|
|
165
|
+
|------|---------|------|------|
|
|
166
|
+
| **InputSpec 定义** | `snake_case` | `max_items` | Pydantic 字段名 |
|
|
167
|
+
| **JSON / params.json** | `camelCase` | `maxItems` | 使用 `alias_generator=to_camel` 时 |
|
|
168
|
+
| **算法访问 ctx.params** | JSON 原始键名 | `ctx.params["maxItems"]` | dict 访问,键名与 JSON 一致 |
|
|
169
|
+
|
|
170
|
+
**InputSpec 定义示例:**
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
# schema/inputspec.py
|
|
174
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
175
|
+
from pydantic.alias_generators import to_camel
|
|
176
|
+
|
|
177
|
+
class MyParams(BaseModel):
|
|
178
|
+
model_config = ConfigDict(
|
|
179
|
+
populate_by_name=True,
|
|
180
|
+
alias_generator=to_camel, # snake_case -> camelCase
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
max_items: int = Field(default=10) # Python: max_items
|
|
184
|
+
dataset_key: str = Field(default="wells") # Python: dataset_key
|
|
185
|
+
|
|
186
|
+
INPUT_SPEC = MyParams
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**对应的 params.json:**
|
|
190
|
+
|
|
191
|
+
```json
|
|
192
|
+
{
|
|
193
|
+
"maxItems": 5,
|
|
194
|
+
"datasetKey": "stages"
|
|
195
|
+
}
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
**算法中访问:**
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
def run(ctx):
|
|
202
|
+
# ctx.params 是 dict,键名与 JSON 一致 (camelCase)
|
|
203
|
+
max_items = ctx.params.get("maxItems", 10)
|
|
204
|
+
dataset_key = ctx.params.get("datasetKey", "wells")
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
> **提示**: 如果 InputSpec 没有配置 `alias_generator`,则 JSON 和 ctx.params 键名都使用 `snake_case`。
|
|
208
|
+
|
|
209
|
+
### 4. 定义输出合约 (OutputContract)
|
|
210
|
+
|
|
211
|
+
创建 `schema/output_contract.py` 声明算法的输出结构:
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
# schema/output_contract.py
|
|
215
|
+
|
|
216
|
+
OUTPUT_CONTRACT = {
|
|
217
|
+
"datasets": [
|
|
218
|
+
{
|
|
219
|
+
"key": "metrics",
|
|
220
|
+
"kind": "scalar",
|
|
221
|
+
"owner": "well",
|
|
222
|
+
"cardinality": "many",
|
|
223
|
+
"required": True,
|
|
224
|
+
"dimensions": ["stage"],
|
|
225
|
+
"schema": {"type": "scalar", "dtype": "float"},
|
|
226
|
+
"role": "primary",
|
|
227
|
+
"description": "每个井/阶段的评估指标"
|
|
228
|
+
},
|
|
229
|
+
{
|
|
230
|
+
"key": "summary",
|
|
231
|
+
"kind": "object",
|
|
232
|
+
"owner": "platform",
|
|
233
|
+
"cardinality": "one",
|
|
234
|
+
"required": True,
|
|
235
|
+
"dimensions": [],
|
|
236
|
+
"schema": {"type": "object"},
|
|
237
|
+
"role": "primary",
|
|
238
|
+
"description": "汇总结果"
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
"key": "debug_plots",
|
|
242
|
+
"kind": "blob",
|
|
243
|
+
"owner": "well",
|
|
244
|
+
"cardinality": "many",
|
|
245
|
+
"required": False,
|
|
246
|
+
"dimensions": [],
|
|
247
|
+
"schema": {"type": "blob", "mime": "image/png"},
|
|
248
|
+
"role": "debug",
|
|
249
|
+
"description": "调试图表 (可选)"
|
|
250
|
+
}
|
|
251
|
+
]
|
|
252
|
+
}
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
#### OutputContract 字段规范
|
|
256
|
+
|
|
257
|
+
| 字段 | 必填 | 类型 | 可选值 | 说明 |
|
|
258
|
+
|------|------|------|--------|------|
|
|
259
|
+
| `key` | **是** | string | - | 数据集唯一键名 |
|
|
260
|
+
| `kind` | **是** | string | `"scalar"` / `"object"` / `"blob"` / `"frame"` | 数据类型 |
|
|
261
|
+
| `owner` | **是** | string | `"stage"` / `"well"` / `"platform"` | 所有者级别 |
|
|
262
|
+
| `cardinality` | 否 | string | `"one"` / `"many"` | 项目数量约束,默认 `"many"` |
|
|
263
|
+
| `required` | 否 | bool | - | 是否必须产出,默认 `true` |
|
|
264
|
+
| `dimensions` | 否 | string[] | - | 维度键列表 |
|
|
265
|
+
| `schema` | **是** | object | - | 数据 schema |
|
|
266
|
+
| `schema.type` | **是** | string | 与 `kind` 对应 | schema 类型标识 |
|
|
267
|
+
| `role` | 否 | string | `"primary"` / `"supporting"` / `"debug"` | 输出角色 |
|
|
268
|
+
| `description` | 否 | string | - | 描述说明 |
|
|
269
|
+
|
|
270
|
+
#### kind 与 schema.type 对应关系
|
|
271
|
+
|
|
272
|
+
| kind | schema.type | ArtifactWriter 方法 | 说明 |
|
|
273
|
+
|------|-------------|---------------------|------|
|
|
274
|
+
| `"scalar"` | `"scalar"` | `write_scalar()` | 标量值 (数字/字符串/布尔) |
|
|
275
|
+
| `"object"` | `"object"` | `write_json()` | JSON 对象 |
|
|
276
|
+
| `"blob"` | `"blob"` | `write_blob()` / `write_file()` | 二进制文件 |
|
|
277
|
+
| `"frame"` | `"frame"` | (暂不支持) | 表格数据 |
|
|
278
|
+
|
|
279
|
+
#### owner 级别说明
|
|
280
|
+
|
|
281
|
+
| owner | 含义 | ArtifactWriter owner 参数 |
|
|
282
|
+
|-------|------|--------------------------|
|
|
283
|
+
| `"platform"` | 平台级 (全局) | `owner={"platformId": "..."}` |
|
|
284
|
+
| `"well"` | 井级 | `owner={"wellId": "..."}` |
|
|
285
|
+
| `"stage"` | 阶段级 | `owner={"stageId": "..."}` |
|
|
286
|
+
|
|
287
|
+
#### cardinality 约束
|
|
288
|
+
|
|
289
|
+
| cardinality | 含义 | 验证规则 |
|
|
290
|
+
|-------------|------|----------|
|
|
291
|
+
| `"one"` | 恰好一个项目 | required=true 时必须 1 项; required=false 时最多 1 项 |
|
|
292
|
+
| `"many"` | 一个或多个 | required=true 时至少 1 项; required=false 时 0 项或多项 |
|
|
293
|
+
|
|
294
|
+
#### dimensions 使用
|
|
295
|
+
|
|
296
|
+
当数据集有维度约束时,写入时必须提供对应的 `dims`:
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
# OutputContract 定义: dimensions: ["stage", "iteration"]
|
|
300
|
+
aw.write_scalar(
|
|
301
|
+
"loss",
|
|
302
|
+
0.05,
|
|
303
|
+
dataset_key="training_metrics",
|
|
304
|
+
owner={"wellId": "W001"},
|
|
305
|
+
dims={"stage": 1, "iteration": 100} # 必须包含所有定义的维度
|
|
306
|
+
)
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### 5. 创建算法清单
|
|
310
|
+
|
|
311
|
+
创建 `manifest.json` — 这是**打包、导入、发布的唯一标准清单**:
|
|
312
|
+
|
|
313
|
+
```json
|
|
314
|
+
{
|
|
315
|
+
"manifestVersion": "1",
|
|
316
|
+
"algorithmId": "my-algorithm",
|
|
317
|
+
"name": "My Algorithm",
|
|
318
|
+
"summary": "算法简短描述 (必填)",
|
|
319
|
+
"notes": "详细说明、使用注意事项等 (可选)",
|
|
320
|
+
"tags": ["analysis", "well-log"],
|
|
321
|
+
"authors": [
|
|
322
|
+
{
|
|
323
|
+
"name": "张三",
|
|
324
|
+
"email": "zhangsan@example.com",
|
|
325
|
+
"organization": "示例公司"
|
|
326
|
+
}
|
|
327
|
+
],
|
|
328
|
+
"contractVersion": "1.0.0",
|
|
329
|
+
"codeVersion": "1.0.0",
|
|
330
|
+
"files": {
|
|
331
|
+
"paramsSchemaPath": "dist/params.schema.json",
|
|
332
|
+
"outputContractPath": "dist/output_contract.json",
|
|
333
|
+
"drsPath": "dist/drs.json"
|
|
334
|
+
},
|
|
335
|
+
"requires": {
|
|
336
|
+
"sdk": "0.1.0",
|
|
337
|
+
"core": "1.0.0"
|
|
338
|
+
},
|
|
339
|
+
"repository": "https://github.com/example/my-algorithm",
|
|
340
|
+
"homepage": "https://example.com/my-algorithm",
|
|
341
|
+
"license": "MIT"
|
|
342
|
+
}
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
#### 字段规范详解
|
|
346
|
+
|
|
347
|
+
| 字段 | 必填 | 类型 | 约束 | 说明 |
|
|
348
|
+
|------|------|------|------|------|
|
|
349
|
+
| `manifestVersion` | **是** | `"1"` | 固定值 | 清单版本 |
|
|
350
|
+
| `algorithmId` | **是** | string | 1-128 字符 | 算法唯一标识符 (用于导入/引用) |
|
|
351
|
+
| `name` | **是** | string | 1-256 字符 | 算法显示名称 |
|
|
352
|
+
| `summary` | **是** | string | 1-256 字符 | 简短描述 |
|
|
353
|
+
| `notes` | 否 | string | - | 详细说明 |
|
|
354
|
+
| `tags` | 否 | string[] | 每项 1-256 字符 | 标签列表 |
|
|
355
|
+
| `authors` | **是** | Author[] | 至少 1 项 | 作者列表 |
|
|
356
|
+
| `authors[].name` | **是** | string | 1-256 字符 | 作者姓名 |
|
|
357
|
+
| `authors[].email` | 否 | string | 3-320 字符 | 邮箱地址 |
|
|
358
|
+
| `authors[].organization` | 否 | string | 1-256 字符 | 所属组织 |
|
|
359
|
+
| `contractVersion` | **是** | string | SemVer 格式 | 输出合约版本 (如 `1.0.0`) |
|
|
360
|
+
| `codeVersion` | **是** | string | - | 代码版本 (用作算法版本标识) |
|
|
361
|
+
| `files` | 否 | object | - | 产物文件路径 (见下表) |
|
|
362
|
+
| `requires` | 否 | object | - | 兼容性要求 |
|
|
363
|
+
| `requires.sdk` | 否 | string | SemVer 格式 | SDK 最低版本 |
|
|
364
|
+
| `requires.core` | 否 | string | SemVer 格式 | Core 最低版本 |
|
|
365
|
+
| `repository` | 否 | string | 1-2048 字符 | 代码仓库 URL |
|
|
366
|
+
| `homepage` | 否 | string | 1-2048 字符 | 主页 URL |
|
|
367
|
+
| `license` | 否 | string | 1-256 字符 | 许可证标识 |
|
|
368
|
+
|
|
369
|
+
#### files 字段详解
|
|
370
|
+
|
|
371
|
+
`files` 用于指定编译产物的位置,导入时 SDK 根据此字段定位文件:
|
|
372
|
+
|
|
373
|
+
| 字段 | 默认值 | 说明 |
|
|
374
|
+
|------|--------|------|
|
|
375
|
+
| `files.paramsSchemaPath` | `"params.schema.json"` | 参数 JSON Schema 路径 |
|
|
376
|
+
| `files.drsPath` | `"drs.json"` | DRS 文件路径 |
|
|
377
|
+
| `files.outputContractPath` | `"output_contract.json"` | 输出合约路径 |
|
|
378
|
+
|
|
379
|
+
**路径规则:**
|
|
380
|
+
- 所有路径均为**相对于算法包根目录**的路径
|
|
381
|
+
- 推荐使用 `dist/` 前缀 (如 `dist/params.schema.json`)
|
|
382
|
+
- 如果省略 `files`,SDK 会在根目录查找默认文件名
|
|
383
|
+
|
|
384
|
+
#### algorithm.json vs manifest.json
|
|
385
|
+
|
|
386
|
+
| 文件 | 状态 | 用途 |
|
|
387
|
+
|------|------|------|
|
|
388
|
+
| `manifest.json` | **标准** | 打包、导入、发布的唯一标准清单 |
|
|
389
|
+
| `algorithm.json` | 辅助 (可选) | 仅开发态辅助文件,供 IDE/工具链使用 |
|
|
390
|
+
|
|
391
|
+
> **建议**: 只维护 `manifest.json`,无需创建 `algorithm.json`。
|
|
392
|
+
|
|
393
|
+
#### 算法包必须包含的文件
|
|
394
|
+
|
|
395
|
+
导入算法包 (zip 或目录) 时,SDK 验证以下文件:
|
|
396
|
+
|
|
397
|
+
| 文件 | 必须 | 说明 |
|
|
398
|
+
|------|------|------|
|
|
399
|
+
| `main.py` | **是** | 算法入口文件,必须包含 `run(ctx)` 函数 |
|
|
400
|
+
| `manifest.json` | **是** | 算法清单 (含 `files.*Path` 字段) |
|
|
401
|
+
| `dist/params.schema.json` | **是** | 参数 JSON Schema (路径由 `files.paramsSchemaPath` 指定) |
|
|
402
|
+
| `dist/drs.json` | **是** | 数据需求规格 (路径由 `files.drsPath` 指定) |
|
|
403
|
+
| `dist/output_contract.json` | **是** | 输出合约 (路径由 `files.outputContractPath` 指定) |
|
|
404
|
+
|
|
405
|
+
> **重要**: 文件实际位置由 `manifest.json` 的 `files.*Path` 字段决定,默认在 `dist/` 目录下。
|
|
406
|
+
|
|
407
|
+
#### 常见导入失败原因
|
|
408
|
+
|
|
409
|
+
1. **`manifest.json not found`**: 包内缺少 manifest.json,或 zip 解压后目录结构嵌套
|
|
410
|
+
2. **`main.py not found`**: 入口文件缺失
|
|
411
|
+
3. **`dist/params.schema.json not found`**: 未执行 `fraclab-sdk algo compile`
|
|
412
|
+
4. **`dist/drs.json not found`**: 编译时未指定 `--bundle` 参数
|
|
413
|
+
5. **`contractVersion must be semver-like`**: contractVersion 格式错误,应为 `x.y.z`
|
|
414
|
+
6. **`authors must contain at least one author`**: authors 列表为空
|
|
415
|
+
|
|
416
|
+
### 6. 项目结构
|
|
417
|
+
|
|
418
|
+
完整的算法工作区结构:
|
|
419
|
+
|
|
420
|
+
```
|
|
421
|
+
my-algorithm/
|
|
422
|
+
├── algorithm.json # 开发时元数据 (可选,编译时使用)
|
|
423
|
+
├── manifest.json # 算法清单 (导出包必须)
|
|
424
|
+
├── main.py # 算法入口 (必须包含 run 函数)
|
|
425
|
+
├── schema/
|
|
426
|
+
│ ├── __init__.py
|
|
427
|
+
│ ├── inputspec.py # INPUT_SPEC 定义
|
|
428
|
+
│ └── output_contract.py # OUTPUT_CONTRACT 定义
|
|
429
|
+
├── lib/ # 可选: 算法依赖模块
|
|
430
|
+
│ └── utils.py
|
|
431
|
+
└── dist/ # 编译产物 (自动生成)
|
|
432
|
+
├── params.schema.json # 从 INPUT_SPEC 编译
|
|
433
|
+
├── output_contract.json # 从 OUTPUT_CONTRACT 编译
|
|
434
|
+
└── drs.json # 从 Bundle 复制
|
|
435
|
+
```
|
|
436
|
+
|
|
437
|
+
**导出后的算法包结构** (zip 内或目录):
|
|
438
|
+
|
|
439
|
+
```
|
|
440
|
+
my-algorithm.zip/
|
|
441
|
+
├── manifest.json # 必须: 算法清单 (含 files.*Path)
|
|
442
|
+
├── main.py # 必须: 入口文件
|
|
443
|
+
├── dist/ # 编译产物目录
|
|
444
|
+
│ ├── params.schema.json # 必须: 参数 Schema
|
|
445
|
+
│ ├── drs.json # 必须: 数据需求规格
|
|
446
|
+
│ └── output_contract.json # 必须: 输出合约
|
|
447
|
+
├── schema/ # 可选: schema 源码
|
|
448
|
+
│ ├── __init__.py
|
|
449
|
+
│ ├── inputspec.py
|
|
450
|
+
│ └── output_contract.py
|
|
451
|
+
└── README.md # 可选: 说明文件
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
---
|
|
455
|
+
|
|
456
|
+
## Bundle 与 Snapshot
|
|
457
|
+
|
|
458
|
+
Bundle 是平台提供的数据包,包含算法所需的输入数据和 DRS 规格。
|
|
459
|
+
|
|
460
|
+
**算法开发者只需知道:**
|
|
461
|
+
|
|
462
|
+
1. **Bundle 由平台/数据团队提供**,用户只需获取路径即可
|
|
463
|
+
2. **不要修改 Bundle 内容** — 任何修改都会导致哈希校验失败
|
|
464
|
+
3. **导入失败时使用验证命令排查**:
|
|
465
|
+
|
|
466
|
+
```bash
|
|
467
|
+
# 验证 Bundle 完整性
|
|
468
|
+
fraclab-sdk validate bundle /path/to/bundle
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
**常见导入错误:**
|
|
472
|
+
- `ds.json hash mismatch`: Bundle 被修改或损坏
|
|
473
|
+
- `drs.json not found`: Bundle 不完整
|
|
474
|
+
- `manifest.json not found`: 非有效 Bundle 目录
|
|
475
|
+
|
|
476
|
+
> 详细的 Bundle 内部结构请参考 [附录 A: Bundle 结构详解](#附录-a-bundle-结构详解)
|
|
477
|
+
|
|
478
|
+
---
|
|
479
|
+
|
|
480
|
+
## 算法开发详解
|
|
481
|
+
|
|
482
|
+
### DataClient - 读取输入数据
|
|
483
|
+
|
|
484
|
+
`DataClient` 提供统一的数据读取接口。
|
|
485
|
+
|
|
486
|
+
```python
|
|
487
|
+
from fraclab_sdk.runtime import DataClient
|
|
488
|
+
from pathlib import Path
|
|
489
|
+
|
|
490
|
+
dc = DataClient(Path("input"))
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
#### 获取数据集信息
|
|
494
|
+
|
|
495
|
+
```python
|
|
496
|
+
# 获取所有数据集键
|
|
497
|
+
keys = dc.get_dataset_keys() # ["wells", "frames", ...]
|
|
498
|
+
|
|
499
|
+
# 获取数据集中的项目数量
|
|
500
|
+
count = dc.get_item_count("wells") # 10
|
|
501
|
+
|
|
502
|
+
# 获取数据集布局类型
|
|
503
|
+
layout = dc.get_layout("wells") # "object_ndjson_lines" 或 "frame_parquet_item_dirs"
|
|
504
|
+
```
|
|
505
|
+
|
|
506
|
+
#### 读取 NDJSON 数据
|
|
507
|
+
|
|
508
|
+
用于 `layout="object_ndjson_lines"` 的数据集:
|
|
509
|
+
|
|
510
|
+
```python
|
|
511
|
+
# 读取单个对象 (按索引)
|
|
512
|
+
obj = dc.read_object("wells", 0) # 返回 dict
|
|
513
|
+
|
|
514
|
+
# 迭代所有对象
|
|
515
|
+
for idx, obj in dc.iterate_objects("wells"):
|
|
516
|
+
print(f"Item {idx}: {obj}")
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
#### 读取 Parquet 数据
|
|
520
|
+
|
|
521
|
+
用于 `layout="frame_parquet_item_dirs"` 的数据集:
|
|
522
|
+
|
|
523
|
+
```python
|
|
524
|
+
# 获取 parquet 文件目录
|
|
525
|
+
parquet_dir = dc.get_parquet_dir("frames", 0)
|
|
526
|
+
|
|
527
|
+
# 获取所有 parquet 文件列表
|
|
528
|
+
parquet_files = dc.get_parquet_files("frames", 0)
|
|
529
|
+
|
|
530
|
+
# 使用 pandas/polars 读取
|
|
531
|
+
import pandas as pd
|
|
532
|
+
df = pd.read_parquet(parquet_dir)
|
|
533
|
+
```
|
|
534
|
+
|
|
535
|
+
### ArtifactWriter - 写入输出结果
|
|
536
|
+
|
|
537
|
+
`ArtifactWriter` 提供安全的输出写入机制,自动防止路径逃逸攻击。
|
|
538
|
+
|
|
539
|
+
```python
|
|
540
|
+
from fraclab_sdk.runtime import ArtifactWriter
|
|
541
|
+
from pathlib import Path
|
|
542
|
+
|
|
543
|
+
aw = ArtifactWriter(Path("output"))
|
|
544
|
+
```
|
|
545
|
+
|
|
546
|
+
#### 写入标量值
|
|
547
|
+
|
|
548
|
+
```python
|
|
549
|
+
# 基本用法
|
|
550
|
+
aw.write_scalar("score", 0.95)
|
|
551
|
+
aw.write_scalar("count", 42)
|
|
552
|
+
aw.write_scalar("name", "result_a")
|
|
553
|
+
|
|
554
|
+
# 指定数据集和所有者
|
|
555
|
+
aw.write_scalar(
|
|
556
|
+
"accuracy",
|
|
557
|
+
0.87,
|
|
558
|
+
dataset_key="metrics",
|
|
559
|
+
owner={"wellId": "W001"},
|
|
560
|
+
dims={"stage": 1},
|
|
561
|
+
meta={"unit": "percent"}
|
|
562
|
+
)
|
|
563
|
+
```
|
|
564
|
+
|
|
565
|
+
#### 写入 JSON
|
|
566
|
+
|
|
567
|
+
```python
|
|
568
|
+
# 基本用法
|
|
569
|
+
path = aw.write_json("metrics", {"accuracy": 0.95, "loss": 0.05})
|
|
570
|
+
|
|
571
|
+
# 自定义文件名
|
|
572
|
+
path = aw.write_json("results", data, filename="analysis_results.json")
|
|
573
|
+
|
|
574
|
+
# 完整参数
|
|
575
|
+
path = aw.write_json(
|
|
576
|
+
"summary",
|
|
577
|
+
{"status": "ok"},
|
|
578
|
+
filename="summary.json",
|
|
579
|
+
dataset_key="outputs",
|
|
580
|
+
owner={"platformId": "P001"}
|
|
581
|
+
)
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
#### 写入二进制文件
|
|
585
|
+
|
|
586
|
+
```python
|
|
587
|
+
# 写入字节数据
|
|
588
|
+
image_bytes = generate_plot()
|
|
589
|
+
path = aw.write_blob(
|
|
590
|
+
"plot",
|
|
591
|
+
image_bytes,
|
|
592
|
+
"plot.png",
|
|
593
|
+
mime_type="image/png"
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
# 复制现有文件
|
|
597
|
+
path = aw.write_file(
|
|
598
|
+
"report",
|
|
599
|
+
Path("/tmp/generated_report.pdf"),
|
|
600
|
+
filename="report.pdf",
|
|
601
|
+
mime_type="application/pdf"
|
|
602
|
+
)
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
### ArtifactWriter 与 OutputContract/Manifest 映射关系
|
|
606
|
+
|
|
607
|
+
ArtifactWriter 的写入操作会自动生成 `output/manifest.json`,理解参数与输出的映射关系是正确使用的关键。
|
|
608
|
+
|
|
609
|
+
#### 参数映射表
|
|
610
|
+
|
|
611
|
+
| ArtifactWriter 参数 | manifest.json 字段 | OutputContract 字段 | 说明 |
|
|
612
|
+
|---------------------|-------------------|---------------------|------|
|
|
613
|
+
| `artifact_key` | `artifact.artifactKey` | - | 制品唯一标识 |
|
|
614
|
+
| `dataset_key` | `datasetKey` | `datasets[].key` | 数据集键,默认 `"artifacts"` |
|
|
615
|
+
| `owner` | `item.owner` | `datasets[].owner` | 所有者: `{wellId, stageId, platformId}` |
|
|
616
|
+
| `dims` | `item.dims` | `datasets[].dimensions` | 维度值字典 |
|
|
617
|
+
| `meta` | `item.meta` | - | 元数据 (manifest 专用) |
|
|
618
|
+
| `item_key` | `item.itemKey` | - | 项目键,默认等于 artifact_key |
|
|
619
|
+
| (写入类型) | `artifact.type` | `datasets[].kind` | 制品类型 |
|
|
620
|
+
|
|
621
|
+
#### 写入操作到 Manifest 的转换
|
|
622
|
+
|
|
623
|
+
```python
|
|
624
|
+
# 算法代码中的写入
|
|
625
|
+
aw.write_scalar(
|
|
626
|
+
"accuracy", # artifact_key
|
|
627
|
+
0.95, # value
|
|
628
|
+
dataset_key="metrics",
|
|
629
|
+
owner={"wellId": "W001"},
|
|
630
|
+
dims={"stage": 1},
|
|
631
|
+
meta={"unit": "percent"}
|
|
632
|
+
)
|
|
633
|
+
```
|
|
634
|
+
|
|
635
|
+
生成的 `output/manifest.json` 片段:
|
|
636
|
+
|
|
637
|
+
```json
|
|
638
|
+
{
|
|
639
|
+
"datasets": [
|
|
640
|
+
{
|
|
641
|
+
"datasetKey": "metrics",
|
|
642
|
+
"items": [
|
|
643
|
+
{
|
|
644
|
+
"itemKey": "accuracy",
|
|
645
|
+
"owner": { "wellId": "W001" },
|
|
646
|
+
"dims": { "stage": 1 },
|
|
647
|
+
"meta": { "unit": "percent" },
|
|
648
|
+
"artifact": {
|
|
649
|
+
"artifactKey": "accuracy",
|
|
650
|
+
"type": "scalar",
|
|
651
|
+
"value": 0.95
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
]
|
|
655
|
+
}
|
|
656
|
+
]
|
|
657
|
+
}
|
|
658
|
+
```
|
|
659
|
+
|
|
660
|
+
#### 类型对应关系
|
|
661
|
+
|
|
662
|
+
| 写入方法 | manifest `artifact.type` | OutputContract `kind` | 说明 |
|
|
663
|
+
|----------|--------------------------|----------------------|------|
|
|
664
|
+
| `write_scalar()` | `"scalar"` | `"scalar"` | 标量值 (直接存 value) |
|
|
665
|
+
| `write_json()` | `"json"` | `"object"` | JSON 对象 (存 uri) |
|
|
666
|
+
| `write_blob()` | `"blob"` | `"blob"` | 二进制文件 (存 uri + mimeType) |
|
|
667
|
+
| `write_file()` | `"blob"` | `"blob"` | 复制文件 (存 uri + mimeType) |
|
|
668
|
+
|
|
669
|
+
#### OutputContract 定义与 ArtifactWriter 使用示例
|
|
670
|
+
|
|
671
|
+
**OutputContract 定义** (`schema/output_contract.py`):
|
|
672
|
+
|
|
673
|
+
```python
|
|
674
|
+
OUTPUT_CONTRACT = {
|
|
675
|
+
"datasets": [
|
|
676
|
+
{
|
|
677
|
+
"key": "metrics",
|
|
678
|
+
"kind": "scalar",
|
|
679
|
+
"owner": "well",
|
|
680
|
+
"cardinality": "many",
|
|
681
|
+
"dimensions": ["stage"],
|
|
682
|
+
"schema": {"type": "scalar", "dtype": "float"}
|
|
683
|
+
},
|
|
684
|
+
{
|
|
685
|
+
"key": "reports",
|
|
686
|
+
"kind": "blob",
|
|
687
|
+
"owner": "well",
|
|
688
|
+
"cardinality": "one",
|
|
689
|
+
"schema": {"type": "blob", "mime": "application/pdf"}
|
|
690
|
+
}
|
|
691
|
+
]
|
|
692
|
+
}
|
|
693
|
+
```
|
|
694
|
+
|
|
695
|
+
**对应的算法写入代码**:
|
|
696
|
+
|
|
697
|
+
```python
|
|
698
|
+
def run(ctx):
|
|
699
|
+
aw = ctx.artifacts
|
|
700
|
+
|
|
701
|
+
# 符合 "metrics" 数据集定义
|
|
702
|
+
# owner="well" → 必须提供 wellId
|
|
703
|
+
# dimensions=["stage"] → dims 必须包含 stage 键
|
|
704
|
+
aw.write_scalar(
|
|
705
|
+
"accuracy",
|
|
706
|
+
0.95,
|
|
707
|
+
dataset_key="metrics",
|
|
708
|
+
owner={"wellId": "W001"},
|
|
709
|
+
dims={"stage": 1}
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
# 符合 "reports" 数据集定义
|
|
713
|
+
# cardinality="one" → 该数据集只能有一个项目
|
|
714
|
+
aw.write_file(
|
|
715
|
+
"report",
|
|
716
|
+
Path("/tmp/report.pdf"),
|
|
717
|
+
dataset_key="reports",
|
|
718
|
+
owner={"wellId": "W001"},
|
|
719
|
+
mime_type="application/pdf"
|
|
720
|
+
)
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
#### 验证输出与合约一致性
|
|
724
|
+
|
|
725
|
+
```bash
|
|
726
|
+
# 验证运行输出是否符合合约
|
|
727
|
+
fraclab-sdk validate run-manifest output/manifest.json --contract dist/output_contract.json
|
|
728
|
+
```
|
|
729
|
+
|
|
730
|
+
验证检查项:
|
|
731
|
+
- 合约中所有 `required=true` 的数据集必须存在
|
|
732
|
+
- 数据集的 `cardinality` 约束 (one/many)
|
|
733
|
+
- `owner` 类型匹配 (well/stage/platform)
|
|
734
|
+
- `dimensions` 键集合匹配
|
|
735
|
+
- `kind` 与 `artifact.type` 兼容
|
|
736
|
+
|
|
737
|
+
### 日志记录
|
|
738
|
+
|
|
739
|
+
使用 `ctx.logger` 记录日志:
|
|
740
|
+
|
|
741
|
+
```python
|
|
742
|
+
def run(ctx):
|
|
743
|
+
logger = ctx.logger
|
|
744
|
+
|
|
745
|
+
logger.debug("调试信息")
|
|
746
|
+
logger.info("常规信息")
|
|
747
|
+
logger.warning("警告信息")
|
|
748
|
+
logger.error("错误信息")
|
|
749
|
+
```
|
|
750
|
+
|
|
751
|
+
日志会同时输出到:
|
|
752
|
+
- 控制台 (INFO 及以上级别)
|
|
753
|
+
- `output/_logs/algorithm.log` 文件 (DEBUG 及以上级别)
|
|
754
|
+
|
|
755
|
+
---
|
|
756
|
+
|
|
757
|
+
## CLI 命令行工具
|
|
758
|
+
|
|
759
|
+
安装后可使用 `fraclab-sdk` 命令。
|
|
760
|
+
|
|
761
|
+
### 运行闭环黄金路径
|
|
762
|
+
|
|
763
|
+
以下是从导入到执行的完整流程示例。
|
|
764
|
+
|
|
765
|
+
#### 1. 导入快照和算法
|
|
766
|
+
|
|
767
|
+
```bash
|
|
768
|
+
# 导入数据快照 (Bundle)
|
|
769
|
+
$ fraclab-sdk snapshot import /path/to/my-bundle
|
|
770
|
+
Imported snapshot: a1b2c3d4
|
|
771
|
+
|
|
772
|
+
# 导入算法包
|
|
773
|
+
$ fraclab-sdk algo import ./my-algorithm.zip
|
|
774
|
+
Imported algorithm: my-algorithm:1.0.0
|
|
775
|
+
```
|
|
776
|
+
|
|
777
|
+
#### 2. 查看已导入资源
|
|
778
|
+
|
|
779
|
+
```bash
|
|
780
|
+
# 列出快照
|
|
781
|
+
$ fraclab-sdk snapshot list
|
|
782
|
+
a1b2c3d4 my-bundle-v1 2024-01-15T10:30:00
|
|
783
|
+
e5f6g7h8 test-bundle 2024-01-14T09:00:00
|
|
784
|
+
|
|
785
|
+
# 列出算法
|
|
786
|
+
$ fraclab-sdk algo list
|
|
787
|
+
my-algorithm 1.0.0 2024-01-15T11:00:00
|
|
788
|
+
other-algo 2.1.0 2024-01-10T08:00:00
|
|
789
|
+
```
|
|
790
|
+
|
|
791
|
+
**ID 格式说明:**
|
|
792
|
+
- `snapshot_id`: 8 位十六进制字符串 (如 `a1b2c3d4`)
|
|
793
|
+
- `algorithm_id`: 算法的 algorithmId 字段值 (如 `my-algorithm`)
|
|
794
|
+
- `version`: 算法的 codeVersion 字段值,遵循 SemVer (如 `1.0.0`)
|
|
795
|
+
|
|
796
|
+
#### 3. 准备参数文件
|
|
797
|
+
|
|
798
|
+
创建 `params.json`:
|
|
799
|
+
|
|
800
|
+
```json
|
|
801
|
+
{
|
|
802
|
+
"threshold": 0.8,
|
|
803
|
+
"debug": false,
|
|
804
|
+
"outputFormat": "detailed",
|
|
805
|
+
"filters": {
|
|
806
|
+
"minDepth": 1000,
|
|
807
|
+
"maxDepth": 5000
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
```
|
|
811
|
+
|
|
812
|
+
**参数文件要求:**
|
|
813
|
+
- JSON 格式,编码 UTF-8
|
|
814
|
+
- 键名对应 InputSpec 中定义的字段
|
|
815
|
+
- 未提供的字段使用 InputSpec 中的默认值
|
|
816
|
+
|
|
817
|
+
#### 4. 创建并执行运行
|
|
818
|
+
|
|
819
|
+
```bash
|
|
820
|
+
# 创建运行 (自动选择所有数据项)
|
|
821
|
+
$ fraclab-sdk run create a1b2c3d4 my-algorithm 1.0.0 --params params.json
|
|
822
|
+
f9e8d7c6
|
|
823
|
+
|
|
824
|
+
# 执行运行
|
|
825
|
+
$ fraclab-sdk run exec f9e8d7c6 --timeout 300
|
|
826
|
+
succeeded (exit_code=0)
|
|
827
|
+
```
|
|
828
|
+
|
|
829
|
+
#### 5. 查看结果
|
|
830
|
+
|
|
831
|
+
```bash
|
|
832
|
+
# 列出产出的制品
|
|
833
|
+
$ fraclab-sdk results list f9e8d7c6
|
|
834
|
+
Status: completed
|
|
835
|
+
accuracy scalar
|
|
836
|
+
summary json file:///Users/.../output/artifacts/summary.json
|
|
837
|
+
report blob file:///Users/.../output/artifacts/report.pdf
|
|
838
|
+
|
|
839
|
+
# 查看运行日志
|
|
840
|
+
$ fraclab-sdk run tail f9e8d7c6
|
|
841
|
+
[INFO] 2024-01-15 12:00:00 - 开始处理,阈值: 0.8
|
|
842
|
+
[INFO] 2024-01-15 12:00:01 - 数据集 wells 包含 3 个项目
|
|
843
|
+
[INFO] 2024-01-15 12:00:05 - 算法执行完成
|
|
844
|
+
|
|
845
|
+
# 查看错误日志 (如有)
|
|
846
|
+
$ fraclab-sdk run tail f9e8d7c6 --stderr
|
|
847
|
+
```
|
|
848
|
+
|
|
849
|
+
#### 6. 运行目录结构
|
|
850
|
+
|
|
851
|
+
执行完成后,`~/.fraclab/runs/<run_id>/` 目录结构:
|
|
852
|
+
|
|
853
|
+
```
|
|
854
|
+
f9e8d7c6/
|
|
855
|
+
├── run_meta.json # 运行元数据
|
|
856
|
+
├── input/ # 输入目录 (物化后的数据)
|
|
857
|
+
│ ├── manifest.json # 输入清单 (含哈希)
|
|
858
|
+
│ ├── ds.json # 运行数据规格 (重新索引)
|
|
859
|
+
│ ├── drs.json # 算法 DRS
|
|
860
|
+
│ ├── params.json # 用户参数
|
|
861
|
+
│ ├── run_context.json # 运行上下文
|
|
862
|
+
│ └── data/ # 数据目录
|
|
863
|
+
│ └── wells/
|
|
864
|
+
│ └── object.ndjson
|
|
865
|
+
└── output/ # 输出目录
|
|
866
|
+
├── manifest.json # 输出清单 ★ 核心结果文件
|
|
867
|
+
├── artifacts/ # 制品文件目录
|
|
868
|
+
│ ├── summary.json
|
|
869
|
+
│ └── report.pdf
|
|
870
|
+
├── _logs/ # 日志目录
|
|
871
|
+
│ ├── stdout.log # 标准输出
|
|
872
|
+
│ ├── stderr.log # 标准错误
|
|
873
|
+
│ ├── algorithm.log # 算法日志 (DEBUG 级别)
|
|
874
|
+
│ └── execute.json # 执行元数据
|
|
875
|
+
```
|
|
876
|
+
|
|
877
|
+
#### 7. 输出 manifest.json 完整示例
|
|
878
|
+
|
|
879
|
+
```json
|
|
880
|
+
{
|
|
881
|
+
"schemaVersion": "1.0.0",
|
|
882
|
+
"run": {
|
|
883
|
+
"runId": "f9e8d7c6",
|
|
884
|
+
"algorithmId": "my-algorithm",
|
|
885
|
+
"contractVersion": "1.0.0",
|
|
886
|
+
"codeVersion": "1.0.0"
|
|
887
|
+
},
|
|
888
|
+
"status": "completed",
|
|
889
|
+
"startedAt": "2024-01-15T12:00:00.000Z",
|
|
890
|
+
"completedAt": "2024-01-15T12:00:05.123Z",
|
|
891
|
+
"datasets": [
|
|
892
|
+
{
|
|
893
|
+
"datasetKey": "artifacts",
|
|
894
|
+
"items": [
|
|
895
|
+
{
|
|
896
|
+
"itemKey": "accuracy",
|
|
897
|
+
"artifact": {
|
|
898
|
+
"artifactKey": "accuracy",
|
|
899
|
+
"type": "scalar",
|
|
900
|
+
"value": 0.95
|
|
901
|
+
}
|
|
902
|
+
},
|
|
903
|
+
{
|
|
904
|
+
"itemKey": "summary",
|
|
905
|
+
"artifact": {
|
|
906
|
+
"artifactKey": "summary",
|
|
907
|
+
"type": "json",
|
|
908
|
+
"mimeType": "application/json",
|
|
909
|
+
"uri": "file:///Users/.../output/artifacts/summary.json"
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
]
|
|
913
|
+
},
|
|
914
|
+
{
|
|
915
|
+
"datasetKey": "reports",
|
|
916
|
+
"items": [
|
|
917
|
+
{
|
|
918
|
+
"itemKey": "report",
|
|
919
|
+
"owner": { "wellId": "W001" },
|
|
920
|
+
"artifact": {
|
|
921
|
+
"artifactKey": "report",
|
|
922
|
+
"type": "blob",
|
|
923
|
+
"mimeType": "application/pdf",
|
|
924
|
+
"uri": "file:///Users/.../output/artifacts/report.pdf"
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
]
|
|
928
|
+
}
|
|
929
|
+
]
|
|
930
|
+
}
|
|
931
|
+
```
|
|
932
|
+
|
|
933
|
+
### 算法开发命令
|
|
934
|
+
|
|
935
|
+
#### 编译算法
|
|
936
|
+
|
|
937
|
+
```bash
|
|
938
|
+
# 编译算法工作区
|
|
939
|
+
fraclab-sdk algo compile ./my-algorithm --bundle /path/to/bundle
|
|
940
|
+
|
|
941
|
+
# 生成:
|
|
942
|
+
# - dist/params.schema.json (从 schema.inputspec:INPUT_SPEC)
|
|
943
|
+
# - dist/output_contract.json (从 schema.output_contract:OUTPUT_CONTRACT)
|
|
944
|
+
# - dist/drs.json (从 bundle 复制)
|
|
945
|
+
```
|
|
946
|
+
|
|
947
|
+
#### 导出算法包
|
|
948
|
+
|
|
949
|
+
```bash
|
|
950
|
+
# 导出为 zip 包
|
|
951
|
+
fraclab-sdk algo export ./my-algorithm ./my-algorithm.zip
|
|
952
|
+
|
|
953
|
+
# 自动编译后导出
|
|
954
|
+
fraclab-sdk algo export ./my-algorithm ./my-algorithm.zip --auto-compile --bundle /path/to/bundle
|
|
955
|
+
```
|
|
956
|
+
|
|
957
|
+
#### 导入算法
|
|
958
|
+
|
|
959
|
+
```bash
|
|
960
|
+
# 导入算法到 SDK 库
|
|
961
|
+
fraclab-sdk algo import ./my-algorithm.zip
|
|
962
|
+
|
|
963
|
+
# 列出已导入的算法
|
|
964
|
+
fraclab-sdk algo list
|
|
965
|
+
```
|
|
966
|
+
|
|
967
|
+
### 验证命令
|
|
968
|
+
|
|
969
|
+
```bash
|
|
970
|
+
# 验证 InputSpec
|
|
971
|
+
fraclab-sdk validate inputspec ./my-algorithm
|
|
972
|
+
|
|
973
|
+
# 验证 OutputContract
|
|
974
|
+
fraclab-sdk validate output-contract ./my-algorithm
|
|
975
|
+
|
|
976
|
+
# 验证 Bundle 完整性
|
|
977
|
+
fraclab-sdk validate bundle /path/to/bundle
|
|
978
|
+
|
|
979
|
+
# 验证运行输出清单
|
|
980
|
+
fraclab-sdk validate run-manifest /path/to/manifest.json --contract /path/to/contract.json
|
|
981
|
+
```
|
|
982
|
+
|
|
983
|
+
### 快照管理命令
|
|
984
|
+
|
|
985
|
+
```bash
|
|
986
|
+
# 导入数据快照
|
|
987
|
+
fraclab-sdk snapshot import /path/to/bundle
|
|
988
|
+
|
|
989
|
+
# 列出已导入快照
|
|
990
|
+
fraclab-sdk snapshot list
|
|
991
|
+
```
|
|
992
|
+
|
|
993
|
+
### 运行管理命令
|
|
994
|
+
|
|
995
|
+
```bash
|
|
996
|
+
# 创建运行
|
|
997
|
+
fraclab-sdk run create <snapshot_id> <algorithm_id> <version> --params params.json
|
|
998
|
+
|
|
999
|
+
# 执行运行
|
|
1000
|
+
fraclab-sdk run exec <run_id> --timeout 300
|
|
1001
|
+
|
|
1002
|
+
# 查看日志
|
|
1003
|
+
fraclab-sdk run tail <run_id>
|
|
1004
|
+
fraclab-sdk run tail <run_id> --stderr
|
|
1005
|
+
```
|
|
1006
|
+
|
|
1007
|
+
### 结果查看命令
|
|
1008
|
+
|
|
1009
|
+
```bash
|
|
1010
|
+
# 列出运行产出的制品
|
|
1011
|
+
fraclab-sdk results list <run_id>
|
|
1012
|
+
```
|
|
1013
|
+
|
|
1014
|
+
### 调试模式
|
|
1015
|
+
|
|
1016
|
+
```bash
|
|
1017
|
+
# 显示完整堆栈跟踪
|
|
1018
|
+
fraclab-sdk --debug <command>
|
|
1019
|
+
```
|
|
1020
|
+
|
|
1021
|
+
---
|
|
1022
|
+
|
|
1023
|
+
## SDK 内部模块
|
|
1024
|
+
|
|
1025
|
+
以下模块供进阶使用或二次开发。
|
|
1026
|
+
|
|
1027
|
+
### SDKConfig - 配置管理
|
|
1028
|
+
|
|
1029
|
+
```python
|
|
1030
|
+
from fraclab_sdk import SDKConfig
|
|
1031
|
+
|
|
1032
|
+
# 使用默认路径 (~/.fraclab)
|
|
1033
|
+
config = SDKConfig()
|
|
1034
|
+
|
|
1035
|
+
# 显式指定路径
|
|
1036
|
+
config = SDKConfig(sdk_home="/custom/path")
|
|
1037
|
+
|
|
1038
|
+
# 通过环境变量: export FRACLAB_SDK_HOME=/custom/path
|
|
1039
|
+
config = SDKConfig() # 自动读取
|
|
1040
|
+
```
|
|
1041
|
+
|
|
1042
|
+
属性:
|
|
1043
|
+
|
|
1044
|
+
| 属性 | 类型 | 说明 |
|
|
1045
|
+
|------|------|------|
|
|
1046
|
+
| `sdk_home` | `Path` | SDK 根目录 |
|
|
1047
|
+
| `snapshots_dir` | `Path` | 快照存储目录 |
|
|
1048
|
+
| `algorithms_dir` | `Path` | 算法存储目录 |
|
|
1049
|
+
| `runs_dir` | `Path` | 运行存储目录 |
|
|
1050
|
+
|
|
1051
|
+
### SnapshotLibrary - 快照管理
|
|
1052
|
+
|
|
1053
|
+
```python
|
|
1054
|
+
from fraclab_sdk import SnapshotLibrary, SDKConfig
|
|
1055
|
+
|
|
1056
|
+
lib = SnapshotLibrary(SDKConfig())
|
|
1057
|
+
|
|
1058
|
+
# 导入快照
|
|
1059
|
+
snapshot_id = lib.import_snapshot("/path/to/bundle")
|
|
1060
|
+
|
|
1061
|
+
# 列出快照
|
|
1062
|
+
snapshots = lib.list_snapshots()
|
|
1063
|
+
|
|
1064
|
+
# 获取快照句柄
|
|
1065
|
+
snapshot = lib.get_snapshot(snapshot_id)
|
|
1066
|
+
|
|
1067
|
+
# 删除快照
|
|
1068
|
+
lib.delete_snapshot(snapshot_id)
|
|
1069
|
+
```
|
|
1070
|
+
|
|
1071
|
+
### SnapshotHandle - 快照访问
|
|
1072
|
+
|
|
1073
|
+
```python
|
|
1074
|
+
snapshot = lib.get_snapshot(snapshot_id)
|
|
1075
|
+
|
|
1076
|
+
# 属性
|
|
1077
|
+
snapshot.directory # Path: 快照目录
|
|
1078
|
+
snapshot.manifest # BundleManifest: 清单
|
|
1079
|
+
snapshot.dataspec # DataSpec: 数据规格
|
|
1080
|
+
snapshot.drs # DRS: 数据需求规格
|
|
1081
|
+
snapshot.data_root # Path: 数据目录
|
|
1082
|
+
|
|
1083
|
+
# 方法
|
|
1084
|
+
datasets = snapshot.get_datasets() # 所有数据集
|
|
1085
|
+
items = snapshot.get_items("wells") # 数据集的所有项目
|
|
1086
|
+
data = snapshot.read_object_line("wells", 0) # 读取 NDJSON 行
|
|
1087
|
+
item_dir = snapshot.get_item_dir("frames", 0) # 获取项目目录
|
|
1088
|
+
parts = snapshot.read_frame_parts("frames", 0) # Parquet 分片列表
|
|
1089
|
+
```
|
|
1090
|
+
|
|
1091
|
+
### AlgorithmLibrary - 算法管理
|
|
1092
|
+
|
|
1093
|
+
```python
|
|
1094
|
+
from fraclab_sdk import AlgorithmLibrary, SDKConfig
|
|
1095
|
+
|
|
1096
|
+
lib = AlgorithmLibrary(SDKConfig())
|
|
1097
|
+
|
|
1098
|
+
# 导入算法
|
|
1099
|
+
algorithm_id, version = lib.import_algorithm("/path/to/algo.zip")
|
|
1100
|
+
|
|
1101
|
+
# 列出算法
|
|
1102
|
+
algorithms = lib.list_algorithms()
|
|
1103
|
+
|
|
1104
|
+
# 获取算法句柄
|
|
1105
|
+
algorithm = lib.get_algorithm(algorithm_id, version)
|
|
1106
|
+
|
|
1107
|
+
# 删除算法
|
|
1108
|
+
lib.delete_algorithm(algorithm_id, version)
|
|
1109
|
+
```
|
|
1110
|
+
|
|
1111
|
+
### AlgorithmHandle - 算法访问
|
|
1112
|
+
|
|
1113
|
+
```python
|
|
1114
|
+
algorithm = lib.get_algorithm(algorithm_id, version)
|
|
1115
|
+
|
|
1116
|
+
# 属性
|
|
1117
|
+
algorithm.directory # Path: 算法目录
|
|
1118
|
+
algorithm.manifest # AlgorithmManifest: 清单
|
|
1119
|
+
algorithm.drs # DRS: 数据需求规格
|
|
1120
|
+
algorithm.params_schema # dict: 参数 JSON Schema
|
|
1121
|
+
algorithm.algorithm_path # Path: main.py 路径
|
|
1122
|
+
```
|
|
1123
|
+
|
|
1124
|
+
### SelectionModel - 数据选择
|
|
1125
|
+
|
|
1126
|
+
```python
|
|
1127
|
+
from fraclab_sdk import SelectionModel
|
|
1128
|
+
|
|
1129
|
+
# 从快照和 DRS 创建选择模型
|
|
1130
|
+
selection = SelectionModel.from_snapshot_and_drs(snapshot, algorithm.drs)
|
|
1131
|
+
|
|
1132
|
+
# 获取可选数据集
|
|
1133
|
+
for ds in selection.get_selectable_datasets():
|
|
1134
|
+
print(f"{ds.dataset_key}: 共 {ds.total_items} 项, 基数要求: {ds.cardinality}")
|
|
1135
|
+
|
|
1136
|
+
# 设置选择
|
|
1137
|
+
selection.set_selected("wells", [0, 1, 2])
|
|
1138
|
+
|
|
1139
|
+
# 获取当前选择
|
|
1140
|
+
selected = selection.get_selected("wells") # [0, 1, 2]
|
|
1141
|
+
|
|
1142
|
+
# 验证选择
|
|
1143
|
+
errors = selection.validate()
|
|
1144
|
+
if not errors:
|
|
1145
|
+
print("选择有效")
|
|
1146
|
+
|
|
1147
|
+
# 构建运行数据规格 (重新索引)
|
|
1148
|
+
run_ds = selection.build_run_ds()
|
|
1149
|
+
|
|
1150
|
+
# 获取索引映射 (run_index -> snapshot_index)
|
|
1151
|
+
mapping = selection.get_selection_mapping("wells")
|
|
1152
|
+
```
|
|
1153
|
+
|
|
1154
|
+
基数规则:
|
|
1155
|
+
- `"one"`: 必须恰好选择 1 个
|
|
1156
|
+
- `"many"`: 必须至少选择 1 个
|
|
1157
|
+
- `"zeroOrMany"`: 可以选择 0 个或多个
|
|
1158
|
+
|
|
1159
|
+
### RunManager - 运行管理
|
|
1160
|
+
|
|
1161
|
+
```python
|
|
1162
|
+
from fraclab_sdk import RunManager, SDKConfig
|
|
1163
|
+
|
|
1164
|
+
mgr = RunManager(SDKConfig())
|
|
1165
|
+
|
|
1166
|
+
# 创建运行
|
|
1167
|
+
run_id = mgr.create_run(
|
|
1168
|
+
snapshot_id=snapshot_id,
|
|
1169
|
+
algorithm_id=algorithm_id,
|
|
1170
|
+
algorithm_version=version,
|
|
1171
|
+
selection=selection,
|
|
1172
|
+
params={"threshold": 0.8}
|
|
1173
|
+
)
|
|
1174
|
+
|
|
1175
|
+
# 执行运行
|
|
1176
|
+
result = mgr.execute(run_id, timeout_s=300)
|
|
1177
|
+
print(f"状态: {result.status}")
|
|
1178
|
+
print(f"退出码: {result.exit_code}")
|
|
1179
|
+
|
|
1180
|
+
# 查询状态
|
|
1181
|
+
status = mgr.get_run_status(run_id)
|
|
1182
|
+
|
|
1183
|
+
# 列出运行
|
|
1184
|
+
runs = mgr.list_runs()
|
|
1185
|
+
|
|
1186
|
+
# 获取运行目录
|
|
1187
|
+
run_dir = mgr.get_run_dir(run_id)
|
|
1188
|
+
|
|
1189
|
+
# 删除运行
|
|
1190
|
+
mgr.delete_run(run_id)
|
|
1191
|
+
```
|
|
1192
|
+
|
|
1193
|
+
### ResultReader - 结果读取
|
|
1194
|
+
|
|
1195
|
+
```python
|
|
1196
|
+
from fraclab_sdk import ResultReader
|
|
1197
|
+
|
|
1198
|
+
reader = ResultReader(run_dir)
|
|
1199
|
+
|
|
1200
|
+
# 检查清单
|
|
1201
|
+
if reader.has_manifest():
|
|
1202
|
+
manifest = reader.read_manifest()
|
|
1203
|
+
print(f"状态: {manifest.status}")
|
|
1204
|
+
|
|
1205
|
+
# 列出制品
|
|
1206
|
+
artifacts = reader.list_artifacts()
|
|
1207
|
+
for art in artifacts:
|
|
1208
|
+
print(f"{art.artifactKey}: {art.type}")
|
|
1209
|
+
|
|
1210
|
+
# 获取制品
|
|
1211
|
+
artifact = reader.get_artifact("score")
|
|
1212
|
+
path = reader.get_artifact_path("metrics")
|
|
1213
|
+
data = reader.read_artifact_json("metrics")
|
|
1214
|
+
value = reader.read_artifact_scalar("score")
|
|
1215
|
+
|
|
1216
|
+
# 读取日志
|
|
1217
|
+
stdout = reader.read_stdout()
|
|
1218
|
+
stderr = reader.read_stderr()
|
|
1219
|
+
algo_log = reader.read_algorithm_log()
|
|
1220
|
+
```
|
|
1221
|
+
|
|
1222
|
+
### Devkit - 开发工具
|
|
1223
|
+
|
|
1224
|
+
```python
|
|
1225
|
+
from fraclab_sdk.devkit import (
|
|
1226
|
+
compile_algorithm,
|
|
1227
|
+
export_algorithm_package,
|
|
1228
|
+
validate_inputspec,
|
|
1229
|
+
validate_output_contract,
|
|
1230
|
+
validate_bundle,
|
|
1231
|
+
validate_run_manifest,
|
|
1232
|
+
)
|
|
1233
|
+
|
|
1234
|
+
# 编译
|
|
1235
|
+
result = compile_algorithm(
|
|
1236
|
+
workspace="/path/to/workspace",
|
|
1237
|
+
bundle_path="/path/to/bundle",
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
# 导出
|
|
1241
|
+
result = export_algorithm_package(
|
|
1242
|
+
workspace="/path/to/workspace",
|
|
1243
|
+
output="/path/to/output.zip",
|
|
1244
|
+
auto_compile=True,
|
|
1245
|
+
bundle_path="/path/to/bundle",
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
# 验证
|
|
1249
|
+
result = validate_inputspec("/path/to/workspace")
|
|
1250
|
+
result = validate_output_contract("/path/to/workspace")
|
|
1251
|
+
result = validate_bundle("/path/to/bundle")
|
|
1252
|
+
result = validate_run_manifest(
|
|
1253
|
+
manifest_path="/path/to/manifest.json",
|
|
1254
|
+
contract_path="/path/to/contract.json"
|
|
1255
|
+
)
|
|
1256
|
+
|
|
1257
|
+
if result.valid:
|
|
1258
|
+
print("验证通过")
|
|
1259
|
+
else:
|
|
1260
|
+
for issue in result.errors:
|
|
1261
|
+
print(f"[{issue.code}] {issue.path}: {issue.message}")
|
|
1262
|
+
```
|
|
1263
|
+
|
|
1264
|
+
---
|
|
1265
|
+
|
|
1266
|
+
## 数据模型
|
|
1267
|
+
|
|
1268
|
+
### DRS (Data Requirement Specification)
|
|
1269
|
+
|
|
1270
|
+
算法对输入数据的需求定义。
|
|
1271
|
+
|
|
1272
|
+
```python
|
|
1273
|
+
from fraclab_sdk.models import DRS
|
|
1274
|
+
|
|
1275
|
+
drs = DRS.model_validate_json(json_string)
|
|
1276
|
+
dataset = drs.get_dataset("wells")
|
|
1277
|
+
print(dataset.cardinality) # "one", "many", "zeroOrMany"
|
|
1278
|
+
```
|
|
1279
|
+
|
|
1280
|
+
### DataSpec
|
|
1281
|
+
|
|
1282
|
+
数据规格定义,描述快照中的数据集结构。
|
|
1283
|
+
|
|
1284
|
+
```python
|
|
1285
|
+
from fraclab_sdk.models import DataSpec
|
|
1286
|
+
|
|
1287
|
+
ds = DataSpec.model_validate_json(json_string)
|
|
1288
|
+
dataset = ds.get_dataset("wells")
|
|
1289
|
+
keys = ds.get_dataset_keys()
|
|
1290
|
+
```
|
|
1291
|
+
|
|
1292
|
+
### OutputContract
|
|
1293
|
+
|
|
1294
|
+
算法输出合约,声明算法产出的数据结构。
|
|
1295
|
+
|
|
1296
|
+
```python
|
|
1297
|
+
from fraclab_sdk.models import OutputContract
|
|
1298
|
+
|
|
1299
|
+
contract = OutputContract.model_validate_json(json_string)
|
|
1300
|
+
dataset = contract.get_dataset("results")
|
|
1301
|
+
artifacts = contract.get_all_artifacts()
|
|
1302
|
+
```
|
|
1303
|
+
|
|
1304
|
+
### RunOutputManifest
|
|
1305
|
+
|
|
1306
|
+
运行输出清单,记录算法执行的结果。
|
|
1307
|
+
|
|
1308
|
+
```python
|
|
1309
|
+
from fraclab_sdk.models import RunOutputManifest
|
|
1310
|
+
|
|
1311
|
+
manifest = RunOutputManifest.model_validate_json(json_string)
|
|
1312
|
+
artifact = manifest.get_artifact("score")
|
|
1313
|
+
all_artifacts = manifest.list_all_artifacts()
|
|
1314
|
+
```
|
|
1315
|
+
|
|
1316
|
+
---
|
|
1317
|
+
|
|
1318
|
+
## 错误处理
|
|
1319
|
+
|
|
1320
|
+
### 异常类型
|
|
1321
|
+
|
|
1322
|
+
```python
|
|
1323
|
+
from fraclab_sdk.errors import (
|
|
1324
|
+
FraclabError, # 基类
|
|
1325
|
+
SnapshotError, # 快照相关
|
|
1326
|
+
HashMismatchError, # 哈希不匹配
|
|
1327
|
+
PathTraversalError, # 路径穿越攻击
|
|
1328
|
+
AlgorithmError, # 算法相关
|
|
1329
|
+
SelectionError, # 选择相关
|
|
1330
|
+
DatasetKeyError, # 数据集键不存在
|
|
1331
|
+
CardinalityError, # 基数验证失败
|
|
1332
|
+
MaterializeError, # 物化错误
|
|
1333
|
+
RunError, # 运行相关
|
|
1334
|
+
TimeoutError, # 执行超时
|
|
1335
|
+
ResultError, # 结果读取
|
|
1336
|
+
OutputContainmentError,# 输出路径逃逸
|
|
1337
|
+
)
|
|
1338
|
+
```
|
|
1339
|
+
|
|
1340
|
+
### 退出码
|
|
1341
|
+
|
|
1342
|
+
```python
|
|
1343
|
+
from fraclab_sdk.errors import ExitCode
|
|
1344
|
+
|
|
1345
|
+
ExitCode.SUCCESS # 0: 成功
|
|
1346
|
+
ExitCode.GENERAL_ERROR # 1: 一般错误
|
|
1347
|
+
ExitCode.INPUT_ERROR # 2: 输入/验证错误
|
|
1348
|
+
ExitCode.RUN_FAILED # 3: 运行失败
|
|
1349
|
+
ExitCode.TIMEOUT # 4: 超时
|
|
1350
|
+
ExitCode.INTERNAL_ERROR # 5: 内部错误
|
|
1351
|
+
```
|
|
1352
|
+
|
|
1353
|
+
### 错误处理示例
|
|
1354
|
+
|
|
1355
|
+
```python
|
|
1356
|
+
from fraclab_sdk.errors import FraclabError, HashMismatchError, CardinalityError
|
|
1357
|
+
|
|
1358
|
+
try:
|
|
1359
|
+
snapshot_id = lib.import_snapshot(path)
|
|
1360
|
+
except HashMismatchError as e:
|
|
1361
|
+
print(f"文件 {e.file_name} 哈希不匹配")
|
|
1362
|
+
print(f"预期: {e.expected}")
|
|
1363
|
+
print(f"实际: {e.actual}")
|
|
1364
|
+
except FraclabError as e:
|
|
1365
|
+
print(f"SDK 错误 (退出码={e.exit_code}): {e}")
|
|
1366
|
+
```
|
|
1367
|
+
|
|
1368
|
+
---
|
|
1369
|
+
|
|
1370
|
+
## 安全特性
|
|
1371
|
+
|
|
1372
|
+
SDK 内置多项安全机制:
|
|
1373
|
+
|
|
1374
|
+
1. **路径穿越防护**: 导入时验证所有路径,拒绝 `..` 和绝对路径
|
|
1375
|
+
2. **哈希验证**: 验证 ds.json 和 drs.json 的 SHA256 哈希
|
|
1376
|
+
3. **输出隔离**: 算法只能写入指定的 output 目录
|
|
1377
|
+
4. **进程隔离**: 算法在子进程中执行,有超时控制
|
|
1378
|
+
5. **原子写入**: 使用 tmp + rename 确保文件完整性
|
|
1379
|
+
|
|
1380
|
+
---
|
|
1381
|
+
|
|
1382
|
+
## 完整示例
|
|
1383
|
+
|
|
1384
|
+
### 算法开发完整流程
|
|
1385
|
+
|
|
1386
|
+
```python
|
|
1387
|
+
# 1. 编写算法
|
|
1388
|
+
# main.py
|
|
1389
|
+
def run(ctx):
|
|
1390
|
+
dc = ctx.data_client
|
|
1391
|
+
aw = ctx.artifacts
|
|
1392
|
+
|
|
1393
|
+
for key in dc.get_dataset_keys():
|
|
1394
|
+
for idx, obj in dc.iterate_objects(key):
|
|
1395
|
+
result = analyze(obj)
|
|
1396
|
+
aw.write_scalar(f"result_{key}_{idx}", result)
|
|
1397
|
+
|
|
1398
|
+
aw.write_json("summary", {"completed": True})
|
|
1399
|
+
|
|
1400
|
+
# 2. 定义规格
|
|
1401
|
+
# schema/inputspec.py
|
|
1402
|
+
from pydantic import BaseModel, Field
|
|
1403
|
+
|
|
1404
|
+
class Params(BaseModel):
|
|
1405
|
+
threshold: float = Field(default=0.5)
|
|
1406
|
+
|
|
1407
|
+
INPUT_SPEC = Params
|
|
1408
|
+
|
|
1409
|
+
# 3. 编译并导出
|
|
1410
|
+
# $ fraclab-sdk algo compile ./my-algo --bundle /path/to/bundle
|
|
1411
|
+
# $ fraclab-sdk algo export ./my-algo ./my-algo.zip
|
|
1412
|
+
```
|
|
1413
|
+
|
|
1414
|
+
### 使用 SDK 执行算法
|
|
1415
|
+
|
|
1416
|
+
```python
|
|
1417
|
+
from fraclab_sdk import (
|
|
1418
|
+
SDKConfig,
|
|
1419
|
+
SnapshotLibrary,
|
|
1420
|
+
AlgorithmLibrary,
|
|
1421
|
+
SelectionModel,
|
|
1422
|
+
RunManager,
|
|
1423
|
+
ResultReader,
|
|
1424
|
+
)
|
|
1425
|
+
|
|
1426
|
+
config = SDKConfig()
|
|
1427
|
+
|
|
1428
|
+
# 导入资源
|
|
1429
|
+
snap_lib = SnapshotLibrary(config)
|
|
1430
|
+
algo_lib = AlgorithmLibrary(config)
|
|
1431
|
+
run_mgr = RunManager(config)
|
|
1432
|
+
|
|
1433
|
+
snapshot_id = snap_lib.import_snapshot("/path/to/bundle")
|
|
1434
|
+
algorithm_id, version = algo_lib.import_algorithm("/path/to/algo.zip")
|
|
1435
|
+
|
|
1436
|
+
# 创建选择
|
|
1437
|
+
snapshot = snap_lib.get_snapshot(snapshot_id)
|
|
1438
|
+
algorithm = algo_lib.get_algorithm(algorithm_id, version)
|
|
1439
|
+
selection = SelectionModel.from_snapshot_and_drs(snapshot, algorithm.drs)
|
|
1440
|
+
selection.set_selected("wells", [0, 1, 2])
|
|
1441
|
+
|
|
1442
|
+
# 执行
|
|
1443
|
+
run_id = run_mgr.create_run(
|
|
1444
|
+
snapshot_id, algorithm_id, version,
|
|
1445
|
+
selection, {"threshold": 0.8}
|
|
1446
|
+
)
|
|
1447
|
+
result = run_mgr.execute(run_id)
|
|
1448
|
+
|
|
1449
|
+
# 读取结果
|
|
1450
|
+
reader = ResultReader(run_mgr.get_run_dir(run_id))
|
|
1451
|
+
for art in reader.list_artifacts():
|
|
1452
|
+
print(f"{art.artifactKey}: {art.type}")
|
|
1453
|
+
```
|
|
1454
|
+
|
|
1455
|
+
---
|
|
1456
|
+
|
|
1457
|
+
## 附录 A: Bundle 结构详解
|
|
1458
|
+
|
|
1459
|
+
> 本节面向平台开发者或需要排查导入问题的用户。普通算法开发者无需了解这些细节。
|
|
1460
|
+
|
|
1461
|
+
### 最小可用 Bundle 目录结构
|
|
1462
|
+
|
|
1463
|
+
```
|
|
1464
|
+
my-bundle/
|
|
1465
|
+
├── manifest.json # 必须: 清单文件 (含哈希校验)
|
|
1466
|
+
├── ds.json # 必须: 数据规格 (DataSpec)
|
|
1467
|
+
├── drs.json # 必须: 数据需求规格 (DRS)
|
|
1468
|
+
└── data/ # 必须: 数据目录
|
|
1469
|
+
└── wells/ # 数据集目录 (datasetKey)
|
|
1470
|
+
└── object.ndjson # NDJSON 格式数据文件
|
|
1471
|
+
```
|
|
1472
|
+
|
|
1473
|
+
### manifest.json 完整示例
|
|
1474
|
+
|
|
1475
|
+
```json
|
|
1476
|
+
{
|
|
1477
|
+
"bundleVersion": "1.0.0",
|
|
1478
|
+
"createdAtUs": 1706000000000000,
|
|
1479
|
+
"specFiles": {
|
|
1480
|
+
"dsPath": "ds.json",
|
|
1481
|
+
"drsPath": "drs.json",
|
|
1482
|
+
"dsSha256": "a1b2c3d4e5f6...(64位十六进制)",
|
|
1483
|
+
"drsSha256": "f6e5d4c3b2a1...(64位十六进制)"
|
|
1484
|
+
},
|
|
1485
|
+
"dataRoot": "data",
|
|
1486
|
+
"datasets": {
|
|
1487
|
+
"wells": {
|
|
1488
|
+
"layout": "object_ndjson_lines",
|
|
1489
|
+
"count": 3
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
```
|
|
1494
|
+
|
|
1495
|
+
**字段说明:**
|
|
1496
|
+
|
|
1497
|
+
| 字段 | 必填 | 说明 |
|
|
1498
|
+
|------|------|------|
|
|
1499
|
+
| `bundleVersion` | 是 | 固定值 `"1.0.0"` |
|
|
1500
|
+
| `createdAtUs` | 是 | 创建时间 (微秒时间戳) |
|
|
1501
|
+
| `specFiles.dsPath` | 是 | ds.json 相对路径,默认 `"ds.json"` |
|
|
1502
|
+
| `specFiles.drsPath` | 是 | drs.json 相对路径,默认 `"drs.json"` |
|
|
1503
|
+
| `specFiles.dsSha256` | 是 | ds.json 的 SHA256 哈希 |
|
|
1504
|
+
| `specFiles.drsSha256` | 是 | drs.json 的 SHA256 哈希 |
|
|
1505
|
+
| `dataRoot` | 是 | 数据目录相对路径,默认 `"data"` |
|
|
1506
|
+
| `datasets` | 是 | 数据集清单,key 为 datasetKey |
|
|
1507
|
+
| `datasets.*.layout` | 是 | `"object_ndjson_lines"` 或 `"frame_parquet_item_dirs"` |
|
|
1508
|
+
| `datasets.*.count` | 是 | 数据集中的项目数量 |
|
|
1509
|
+
|
|
1510
|
+
### ds.json (DataSpec) 完整示例
|
|
1511
|
+
|
|
1512
|
+
```json
|
|
1513
|
+
{
|
|
1514
|
+
"schemaVersion": "1.0.0",
|
|
1515
|
+
"datasets": [
|
|
1516
|
+
{
|
|
1517
|
+
"key": "wells",
|
|
1518
|
+
"resourceType": "well",
|
|
1519
|
+
"layout": "object_ndjson_lines",
|
|
1520
|
+
"items": [
|
|
1521
|
+
{ "owner": { "wellId": "W001" } },
|
|
1522
|
+
{ "owner": { "wellId": "W002" } },
|
|
1523
|
+
{ "owner": { "wellId": "W003" } }
|
|
1524
|
+
]
|
|
1525
|
+
}
|
|
1526
|
+
]
|
|
1527
|
+
}
|
|
1528
|
+
```
|
|
1529
|
+
|
|
1530
|
+
**字段说明:**
|
|
1531
|
+
|
|
1532
|
+
| 字段 | 必填 | 说明 |
|
|
1533
|
+
|------|------|------|
|
|
1534
|
+
| `datasets[].key` | 是 | 数据集键名,对应 data/ 下的目录名 |
|
|
1535
|
+
| `datasets[].resourceType` | 否 | 资源类型标识 |
|
|
1536
|
+
| `datasets[].layout` | 是 | 数据布局,决定数据存储格式 |
|
|
1537
|
+
| `datasets[].items` | 是 | 项目列表,每个元素对应一条数据 |
|
|
1538
|
+
| `datasets[].items[].owner` | 否 | 所有者标识,包含 platformId/wellId/stageId |
|
|
1539
|
+
|
|
1540
|
+
### drs.json (DRS) 完整示例
|
|
1541
|
+
|
|
1542
|
+
```json
|
|
1543
|
+
{
|
|
1544
|
+
"schemaVersion": "1.0.0",
|
|
1545
|
+
"datasets": [
|
|
1546
|
+
{
|
|
1547
|
+
"key": "wells",
|
|
1548
|
+
"resource": "well",
|
|
1549
|
+
"cardinality": "many",
|
|
1550
|
+
"description": "井数据集"
|
|
1551
|
+
}
|
|
1552
|
+
]
|
|
1553
|
+
}
|
|
1554
|
+
```
|
|
1555
|
+
|
|
1556
|
+
**字段说明:**
|
|
1557
|
+
|
|
1558
|
+
| 字段 | 必填 | 说明 |
|
|
1559
|
+
|------|------|------|
|
|
1560
|
+
| `datasets[].key` | 是 | 数据集键名,必须与 ds.json 中的 key 匹配 |
|
|
1561
|
+
| `datasets[].resource` | 否 | 资源类型 (别名 `resourceType`) |
|
|
1562
|
+
| `datasets[].cardinality` | 是 | 基数要求: `"one"` / `"many"` / `"zeroOrMany"` |
|
|
1563
|
+
| `datasets[].description` | 否 | 数据集描述 |
|
|
1564
|
+
|
|
1565
|
+
**基数规则:**
|
|
1566
|
+
- `"one"`: 必须恰好选择 1 个项目
|
|
1567
|
+
- `"many"`: 必须至少选择 1 个项目
|
|
1568
|
+
- `"zeroOrMany"`: 可以选择 0 个或多个项目
|
|
1569
|
+
|
|
1570
|
+
### 数据目录布局
|
|
1571
|
+
|
|
1572
|
+
**布局 1: object_ndjson_lines (NDJSON 格式)**
|
|
1573
|
+
|
|
1574
|
+
```
|
|
1575
|
+
data/
|
|
1576
|
+
└── wells/
|
|
1577
|
+
├── object.ndjson # 必须: 每行一个 JSON 对象
|
|
1578
|
+
└── object.idx.u64 # 可选: 索引文件 (加速随机访问)
|
|
1579
|
+
```
|
|
1580
|
+
|
|
1581
|
+
`object.ndjson` 示例 (每行一个 JSON 对象):
|
|
1582
|
+
```
|
|
1583
|
+
{"wellId": "W001", "name": "Well Alpha", "depth": 3000}
|
|
1584
|
+
{"wellId": "W002", "name": "Well Beta", "depth": 3500}
|
|
1585
|
+
{"wellId": "W003", "name": "Well Gamma", "depth": 2800}
|
|
1586
|
+
```
|
|
1587
|
+
|
|
1588
|
+
**布局 2: frame_parquet_item_dirs (Parquet 格式)**
|
|
1589
|
+
|
|
1590
|
+
```
|
|
1591
|
+
data/
|
|
1592
|
+
└── frames/
|
|
1593
|
+
└── parquet/
|
|
1594
|
+
├── item-00000/ # 第 0 个项目
|
|
1595
|
+
│ └── data.parquet
|
|
1596
|
+
├── item-00001/ # 第 1 个项目
|
|
1597
|
+
│ └── data.parquet
|
|
1598
|
+
└── item-00002/ # 第 2 个项目
|
|
1599
|
+
└── data.parquet
|
|
1600
|
+
```
|
|
1601
|
+
|
|
1602
|
+
目录命名规则: `item-{index:05d}` (5 位数字,前导零填充)
|
|
1603
|
+
|
|
1604
|
+
### 生成哈希值
|
|
1605
|
+
|
|
1606
|
+
使用 Python 计算 SHA256 哈希:
|
|
1607
|
+
|
|
1608
|
+
```python
|
|
1609
|
+
import hashlib
|
|
1610
|
+
from pathlib import Path
|
|
1611
|
+
|
|
1612
|
+
def compute_sha256(file_path: str) -> str:
|
|
1613
|
+
content = Path(file_path).read_bytes()
|
|
1614
|
+
return hashlib.sha256(content).hexdigest()
|
|
1615
|
+
|
|
1616
|
+
# 示例
|
|
1617
|
+
ds_hash = compute_sha256("ds.json")
|
|
1618
|
+
drs_hash = compute_sha256("drs.json")
|
|
1619
|
+
print(f"dsSha256: {ds_hash}")
|
|
1620
|
+
print(f"drsSha256: {drs_hash}")
|
|
1621
|
+
```
|
|
1622
|
+
|