sci-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sci_agent-0.2.0.dist-info/METADATA +11 -0
- sci_agent-0.2.0.dist-info/RECORD +32 -0
- sci_agent-0.2.0.dist-info/WHEEL +5 -0
- sci_agent-0.2.0.dist-info/entry_points.txt +2 -0
- sci_agent-0.2.0.dist-info/top_level.txt +1 -0
- simple_ci/__init__.py +1 -0
- simple_ci/__main__.py +42 -0
- simple_ci/actions/__init__.py +136 -0
- simple_ci/actions/base.py +318 -0
- simple_ci/actions/build.py +31 -0
- simple_ci/actions/deploy.py +23 -0
- simple_ci/actions/hooks.py +153 -0
- simple_ci/actions/image.py +28 -0
- simple_ci/actions/source.py +32 -0
- simple_ci/actions/target.py +63 -0
- simple_ci/cli/__init__.py +1 -0
- simple_ci/cli/client.py +104 -0
- simple_ci/cli/config.py +276 -0
- simple_ci/cli/history.py +82 -0
- simple_ci/cli/main.py +333 -0
- simple_ci/cli/render.py +111 -0
- simple_ci/errors.py +37 -0
- simple_ci/models.py +134 -0
- simple_ci/server/__init__.py +1 -0
- simple_ci/server/app.py +118 -0
- simple_ci/server/auth.py +40 -0
- simple_ci/server/executor.py +304 -0
- simple_ci/server/main.py +37 -0
- simple_ci/server/store.py +152 -0
- simple_ci/ui/__init__.py +1 -0
- simple_ci/ui/server.py +285 -0
- simple_ci/ui/static/index.html +271 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Python Hook 查找、加载与结果脱敏工具。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
from copy import deepcopy
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from types import ModuleType
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from simple_ci.actions.base import collect_secret_values, redact_secrets
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def run_hook(hook_name: str, context: dict[str, Any]) -> dict[str, Any]:
|
|
15
|
+
"""按流水线、项目、全局顺序运行 Python Hook,并返回可序列化的字典结果。
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
hook_name: 不含 ``.py`` 后缀的 Hook 名称。
|
|
19
|
+
context: 传递给 Hook ``run(context)`` 的上下文,至少包含 args、target、
|
|
20
|
+
project、pipeline、step、run_id 等执行信息。
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
Hook 返回的字典;加载失败、缺少 run 函数、异常或返回非字典时统一转换为
|
|
24
|
+
``status=failed`` 且 ``exit_code=1`` 的失败结果。所有结果都会基于上下文中
|
|
25
|
+
的敏感字段做脱敏,避免 Hook 异常泄露 token/password 等凭据。
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
secret_values = collect_secret_values(context)
|
|
29
|
+
safe_context = _sanitize_value(context, secret_values)
|
|
30
|
+
hook_path = _find_hook_path(hook_name, context)
|
|
31
|
+
if hook_path is None:
|
|
32
|
+
return _failure(f"Hook not found: {hook_name}", secret_values)
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
module = _load_hook_module(hook_name, hook_path)
|
|
36
|
+
run = getattr(module, "run", None)
|
|
37
|
+
if not callable(run):
|
|
38
|
+
return _failure(f"Hook {hook_name} must define run(context).", secret_values)
|
|
39
|
+
result = run(safe_context)
|
|
40
|
+
if not isinstance(result, dict):
|
|
41
|
+
return _failure(f"Hook {hook_name} run(context) must return a dict.", secret_values)
|
|
42
|
+
normalized = deepcopy(result)
|
|
43
|
+
if normalized.get("status") == "failed":
|
|
44
|
+
normalized["exit_code"] = 1
|
|
45
|
+
return _sanitize_value(normalized, secret_values)
|
|
46
|
+
except Exception as exc: # noqa: BLE001 - Hook 是用户脚本边界,必须转换为失败结果。
|
|
47
|
+
return _failure(str(exc), secret_values)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _find_hook_path(hook_name: str, context: dict[str, Any]) -> Path | None:
|
|
51
|
+
"""按 pipeline hooks > project hooks > global hooks 顺序查找 Hook 文件。"""
|
|
52
|
+
|
|
53
|
+
if not _is_safe_basename(hook_name):
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
project = str(context.get("project") or "")
|
|
57
|
+
pipeline = str(context.get("pipeline") or context.get("pipeline_name") or "")
|
|
58
|
+
if not _is_safe_context_segment(project) or not _is_safe_context_segment(pipeline):
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
candidates = (
|
|
62
|
+
(Path("sci") / "projects" / project / "pipeline" / pipeline / "hooks", f"{hook_name}.py"),
|
|
63
|
+
(Path("sci") / "projects" / project / "hooks", f"{hook_name}.py"),
|
|
64
|
+
(Path("sci") / "hooks", f"{hook_name}.py"),
|
|
65
|
+
)
|
|
66
|
+
for hooks_root, filename in candidates:
|
|
67
|
+
candidate = hooks_root / filename
|
|
68
|
+
if _is_within_hooks_root(candidate, hooks_root) and candidate.is_file():
|
|
69
|
+
return candidate
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _is_safe_basename(value: str) -> bool:
|
|
74
|
+
"""校验 Hook 名称只能是安全 basename,禁止绝对路径、分隔符和 ``..``。"""
|
|
75
|
+
|
|
76
|
+
path = Path(value)
|
|
77
|
+
return (
|
|
78
|
+
bool(value)
|
|
79
|
+
and not path.is_absolute()
|
|
80
|
+
and path.name == value
|
|
81
|
+
and "/" not in value
|
|
82
|
+
and "\\" not in value
|
|
83
|
+
and ".." not in value
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _is_safe_context_segment(value: str) -> bool:
|
|
88
|
+
"""校验 project/pipeline 只能是单段路径名,避免目录穿越。"""
|
|
89
|
+
|
|
90
|
+
if not value:
|
|
91
|
+
return False
|
|
92
|
+
path = Path(value)
|
|
93
|
+
return (
|
|
94
|
+
not path.is_absolute()
|
|
95
|
+
and path.name == value
|
|
96
|
+
and "/" not in value
|
|
97
|
+
and "\\" not in value
|
|
98
|
+
and ".." not in value
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _is_within_hooks_root(candidate: Path, hooks_root: Path) -> bool:
|
|
103
|
+
"""确认解析后的候选文件仍位于对应 hooks 根目录下。"""
|
|
104
|
+
|
|
105
|
+
try:
|
|
106
|
+
candidate.resolve().relative_to(hooks_root.resolve())
|
|
107
|
+
except ValueError:
|
|
108
|
+
return False
|
|
109
|
+
return True
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _load_hook_module(hook_name: str, hook_path: Path) -> ModuleType:
|
|
113
|
+
"""使用 importlib 按文件路径加载 Hook,避免依赖 Hook 目录成为 Python 包。"""
|
|
114
|
+
|
|
115
|
+
module_name = f"simple_ci_user_hook_{hook_name}_{abs(hash(str(hook_path.resolve())))}"
|
|
116
|
+
spec = importlib.util.spec_from_file_location(module_name, hook_path)
|
|
117
|
+
if spec is None or spec.loader is None:
|
|
118
|
+
raise RuntimeError(f"Unable to load hook: {hook_name}")
|
|
119
|
+
module = importlib.util.module_from_spec(spec)
|
|
120
|
+
spec.loader.exec_module(module)
|
|
121
|
+
return module
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _failure(error: str, secret_values: set[str]) -> dict[str, Any]:
|
|
125
|
+
"""构造标准 Hook 失败结果,并对错误文本做脱敏。"""
|
|
126
|
+
|
|
127
|
+
safe_error = redact_secrets(error, secret_values)
|
|
128
|
+
return {
|
|
129
|
+
"status": "failed",
|
|
130
|
+
"exit_code": 1,
|
|
131
|
+
"stdout_tail": "",
|
|
132
|
+
"stderr_tail": safe_error,
|
|
133
|
+
"error": safe_error,
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _sanitize_value(value: Any, secret_values: set[str]) -> Any:
|
|
138
|
+
"""递归脱敏 Hook 返回结果中的敏感字段和敏感值。"""
|
|
139
|
+
|
|
140
|
+
if isinstance(value, dict):
|
|
141
|
+
sanitized: dict[str, Any] = {}
|
|
142
|
+
for key, child in value.items():
|
|
143
|
+
key_text = str(key)
|
|
144
|
+
if any(part in key_text.lower() for part in ("token", "secret", "password", "credential")):
|
|
145
|
+
sanitized[key_text] = "***"
|
|
146
|
+
else:
|
|
147
|
+
sanitized[key_text] = _sanitize_value(child, secret_values)
|
|
148
|
+
return sanitized
|
|
149
|
+
if isinstance(value, list):
|
|
150
|
+
return [_sanitize_value(item, secret_values) for item in value]
|
|
151
|
+
if isinstance(value, str):
|
|
152
|
+
return redact_secrets(value, secret_values)
|
|
153
|
+
return deepcopy(value)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""镜像相关内置动作的命令构造。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def build_image_command(params: dict[str, Any]) -> list[str]:
|
|
9
|
+
"""构造 docker build 命令,保留参数顺序以便测试和日志稳定。"""
|
|
10
|
+
|
|
11
|
+
image = str(params["image"])
|
|
12
|
+
tag = str(params.get("tag", "latest"))
|
|
13
|
+
command = ["docker", "build", str(params.get("context", "."))]
|
|
14
|
+
dockerfile = params.get("dockerfile")
|
|
15
|
+
if dockerfile:
|
|
16
|
+
command.extend(["-f", str(dockerfile)])
|
|
17
|
+
command.extend(["-t", f"{image}:{tag}"])
|
|
18
|
+
for key, value in (params.get("build_args") or {}).items():
|
|
19
|
+
command.extend(["--build-arg", f"{key}={value}"])
|
|
20
|
+
return command
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def build_push_command(params: dict[str, Any]) -> list[str]:
|
|
24
|
+
"""构造 docker push 命令。"""
|
|
25
|
+
|
|
26
|
+
image = str(params["image"])
|
|
27
|
+
tag = str(params.get("tag", "latest"))
|
|
28
|
+
return ["docker", "push", f"{image}:{tag}"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""源码相关内置动作的命令构造。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from simple_ci.actions.base import inject_basic_auth
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build_git_pull_command(params: dict[str, Any]) -> list[str] | list[list[str]]:
|
|
12
|
+
"""构造 git 拉取命令;目录已有 .git 时执行 fetch/reset,否则首次 clone。"""
|
|
13
|
+
|
|
14
|
+
original_repo = str(params["repo"])
|
|
15
|
+
repo = inject_basic_auth(original_repo, params.get("credential"))
|
|
16
|
+
branch = str(params.get("branch", "main"))
|
|
17
|
+
directory = str(params.get("directory") or params.get("checkout_dir") or ".")
|
|
18
|
+
checkout_path = Path(directory)
|
|
19
|
+
if not checkout_path.is_absolute():
|
|
20
|
+
base_dir = Path(str(params.get("cwd") or params.get("workdir") or "."))
|
|
21
|
+
checkout_path = base_dir / checkout_path
|
|
22
|
+
if (checkout_path / ".git").exists():
|
|
23
|
+
if repo != original_repo:
|
|
24
|
+
return [
|
|
25
|
+
["git", "-C", directory, "fetch", repo, branch],
|
|
26
|
+
["git", "-C", directory, "reset", "--hard", "FETCH_HEAD"],
|
|
27
|
+
]
|
|
28
|
+
return [
|
|
29
|
+
["git", "-C", directory, "fetch", "origin", branch],
|
|
30
|
+
["git", "-C", directory, "reset", "--hard", f"origin/{branch}"],
|
|
31
|
+
]
|
|
32
|
+
return ["git", "clone", "--branch", branch, repo, directory]
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""目标主机相关内置动作的命令构造。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def build_target_shell_command(params: dict[str, Any], payload: dict[str, Any] | None = None) -> list[str] | str:
|
|
10
|
+
"""返回目标 shell 命令,支持 inline command 与 script+args 两种模式。
|
|
11
|
+
|
|
12
|
+
当配置 ``script`` 时,按流水线目录、项目目录、全局目录三层查找;script
|
|
13
|
+
名称拒绝绝对路径和 ``..``,避免越权执行工作区外文件。未配置 script 时
|
|
14
|
+
保持原有 inline ``command`` 能力。
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
if params.get("script"):
|
|
18
|
+
return _build_script_command(params, payload or {})
|
|
19
|
+
command = params.get("command")
|
|
20
|
+
if not isinstance(command, (str, list)):
|
|
21
|
+
raise ValueError("Missing target shell command.")
|
|
22
|
+
return command
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _build_script_command(params: dict[str, Any], payload: dict[str, Any]) -> list[str]:
|
|
26
|
+
"""解析 script 路径并拼接 argv 参数。"""
|
|
27
|
+
|
|
28
|
+
script_name = str(params["script"])
|
|
29
|
+
_validate_script_name(script_name)
|
|
30
|
+
root = Path(str(params.get("scripts_root") or payload.get("workspace_root") or params.get("cwd") or "."))
|
|
31
|
+
project_id = str(params.get("project") or payload.get("project") or "")
|
|
32
|
+
pipeline_id = str(params.get("pipeline") or payload.get("pipeline") or "")
|
|
33
|
+
candidates = [
|
|
34
|
+
root / "sci" / "projects" / project_id / "pipeline" / pipeline_id / "scripts" / script_name,
|
|
35
|
+
root / "sci" / "projects" / project_id / "scripts" / script_name,
|
|
36
|
+
root / "sci" / "scripts" / script_name,
|
|
37
|
+
]
|
|
38
|
+
for candidate in candidates:
|
|
39
|
+
if candidate.is_file():
|
|
40
|
+
return [str(candidate), *_normalize_args(params.get("args", []))]
|
|
41
|
+
raise ValueError(f"target shell script not found: {script_name}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _validate_script_name(script_name: str) -> None:
|
|
45
|
+
"""校验 script 名称只能是工作区内相对路径。"""
|
|
46
|
+
|
|
47
|
+
path = Path(script_name)
|
|
48
|
+
if (
|
|
49
|
+
path.is_absolute()
|
|
50
|
+
or script_name.startswith(("/", "\\"))
|
|
51
|
+
or any(part == ".." for part in path.parts)
|
|
52
|
+
):
|
|
53
|
+
raise ValueError(f"unsafe target shell script: {script_name}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _normalize_args(value: Any) -> list[str]:
|
|
57
|
+
"""把 args 规范为字符串 argv 列表;单个标量按一个参数处理。"""
|
|
58
|
+
|
|
59
|
+
if value in (None, ""):
|
|
60
|
+
return []
|
|
61
|
+
if isinstance(value, list):
|
|
62
|
+
return [str(item) for item in value]
|
|
63
|
+
return [str(value)]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Command-line handlers for sci-agent."""
|
simple_ci/cli/client.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
TERMINAL_STATUSES = {"success", "failed", "cancelled"}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ServerApiError(RuntimeError):
|
|
13
|
+
"""表示 sci-agent server API 返回非成功响应或客户端处理失败。"""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SimpleCiClient:
|
|
17
|
+
"""封装 sci-agent server HTTP API,供 CLI 以统一方式提交和查询运行。"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, base_url: str, token: str, transport: httpx.BaseTransport | None = None) -> None:
|
|
20
|
+
self.base_url = base_url.rstrip("/") + "/"
|
|
21
|
+
self.token = token
|
|
22
|
+
self._client = httpx.Client(
|
|
23
|
+
base_url=self.base_url,
|
|
24
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
25
|
+
transport=transport,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
def __enter__(self) -> "SimpleCiClient":
|
|
29
|
+
"""进入上下文管理器,返回当前客户端实例。"""
|
|
30
|
+
|
|
31
|
+
return self
|
|
32
|
+
|
|
33
|
+
def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
|
|
34
|
+
"""退出上下文管理器时关闭底层 httpx.Client。"""
|
|
35
|
+
|
|
36
|
+
self.close()
|
|
37
|
+
|
|
38
|
+
def post_run(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
39
|
+
"""提交流水线运行请求,并返回服务端创建的 run 快照。"""
|
|
40
|
+
|
|
41
|
+
return self._request_json("POST", "/api/v1/pipeline/run", json=payload)
|
|
42
|
+
|
|
43
|
+
def get_run(self, run_id: str) -> dict[str, Any]:
|
|
44
|
+
"""查询指定 run_id 的当前状态快照。"""
|
|
45
|
+
|
|
46
|
+
return self._request_json("GET", f"/api/v1/runs/{run_id}")
|
|
47
|
+
|
|
48
|
+
def get_logs(self, run_id: str) -> dict[str, Any]:
|
|
49
|
+
"""查询指定 run_id 的日志结构。"""
|
|
50
|
+
|
|
51
|
+
return self._request_json("GET", f"/api/v1/runs/{run_id}/logs")
|
|
52
|
+
|
|
53
|
+
def run_and_poll(
|
|
54
|
+
self,
|
|
55
|
+
payload: dict[str, Any],
|
|
56
|
+
interval_seconds: float = 5,
|
|
57
|
+
timeout_seconds: float = 1800,
|
|
58
|
+
) -> dict[str, Any]:
|
|
59
|
+
"""提交流水线并轮询,直到进入终态或超时。"""
|
|
60
|
+
|
|
61
|
+
deadline = time.monotonic() + timeout_seconds
|
|
62
|
+
run = self.post_run(payload)
|
|
63
|
+
run_id = str(run["run_id"])
|
|
64
|
+
while not self._is_terminal(run):
|
|
65
|
+
if time.monotonic() >= deadline:
|
|
66
|
+
raise TimeoutError(f"Timed out waiting for run {run_id}")
|
|
67
|
+
if interval_seconds > 0:
|
|
68
|
+
time.sleep(min(interval_seconds, max(0.0, deadline - time.monotonic())))
|
|
69
|
+
if time.monotonic() >= deadline:
|
|
70
|
+
raise TimeoutError(f"Timed out waiting for run {run_id}")
|
|
71
|
+
run = self.get_run(run_id)
|
|
72
|
+
return run
|
|
73
|
+
|
|
74
|
+
def close(self) -> None:
|
|
75
|
+
"""关闭底层 httpx.Client,释放连接池资源。"""
|
|
76
|
+
|
|
77
|
+
self._client.close()
|
|
78
|
+
|
|
79
|
+
def _request_json(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
|
|
80
|
+
"""发送请求并统一处理错误,异常消息会脱敏 token 以避免泄露凭据。"""
|
|
81
|
+
|
|
82
|
+
response = self._client.request(method, path, **kwargs)
|
|
83
|
+
if not response.is_success:
|
|
84
|
+
raise ServerApiError(
|
|
85
|
+
f"Server API request failed with status {response.status_code}: "
|
|
86
|
+
f"{self._sanitize(response.text)}"
|
|
87
|
+
)
|
|
88
|
+
data = response.json()
|
|
89
|
+
if not isinstance(data, dict):
|
|
90
|
+
raise ServerApiError("Server API returned a non-object JSON response")
|
|
91
|
+
return data
|
|
92
|
+
|
|
93
|
+
def _sanitize(self, value: str) -> str:
|
|
94
|
+
"""从错误文本中移除当前客户端 token,保证日志和异常不会暴露认证信息。"""
|
|
95
|
+
|
|
96
|
+
if not self.token:
|
|
97
|
+
return value
|
|
98
|
+
return value.replace(self.token, "***")
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def _is_terminal(run: dict[str, Any]) -> bool:
|
|
102
|
+
"""判断 run 是否进入终态;未知状态按非终态处理以便继续轮询。"""
|
|
103
|
+
|
|
104
|
+
return str(run.get("status")) in TERMINAL_STATUSES
|
simple_ci/cli/config.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""Simple CI 工作区配置加载器。
|
|
2
|
+
|
|
3
|
+
负责把 ``sci/`` 目录下的 TOML/YAML 配置和根目录 ``.env`` 合并为领域模型。
|
|
4
|
+
所有敏感值只用于填充模型字段,错误信息不得输出实际凭据内容。
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import tomllib
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Mapping
|
|
15
|
+
|
|
16
|
+
import yaml
|
|
17
|
+
from dotenv import dotenv_values
|
|
18
|
+
|
|
19
|
+
from simple_ci.errors import ConfigError
|
|
20
|
+
from simple_ci.models import Credential, Pipeline, Project, ProjectSource, Step, Target, TargetServer
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Workspace:
|
|
25
|
+
"""Simple CI 工作区配置集合。"""
|
|
26
|
+
|
|
27
|
+
root: Path
|
|
28
|
+
targets: dict[str, Target]
|
|
29
|
+
credentials: dict[str, Credential]
|
|
30
|
+
projects: dict[str, Project]
|
|
31
|
+
pipelines: dict[str, Pipeline]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_workspace(root: str | Path) -> Workspace:
|
|
35
|
+
"""从工作区根目录加载 ``sci`` 配置。"""
|
|
36
|
+
|
|
37
|
+
root_path = Path(root)
|
|
38
|
+
sci_dir = root_path / "sci"
|
|
39
|
+
if not sci_dir.is_dir():
|
|
40
|
+
raise ConfigError(f"missing sci directory: {sci_dir}")
|
|
41
|
+
|
|
42
|
+
env = _load_env(root_path)
|
|
43
|
+
credentials = _load_credentials(sci_dir / "config.toml", env)
|
|
44
|
+
targets = _load_targets(sci_dir / "targets", env)
|
|
45
|
+
projects, pipelines = _load_projects(sci_dir / "projects", credentials)
|
|
46
|
+
|
|
47
|
+
return Workspace(
|
|
48
|
+
root=root_path,
|
|
49
|
+
targets=targets,
|
|
50
|
+
credentials=credentials,
|
|
51
|
+
projects=projects,
|
|
52
|
+
pipelines=pipelines,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _load_env(root: Path) -> dict[str, str]:
|
|
57
|
+
values: dict[str, str] = {}
|
|
58
|
+
env_file = root / ".env"
|
|
59
|
+
if env_file.exists():
|
|
60
|
+
for key, value in dotenv_values(env_file, encoding="utf-8").items():
|
|
61
|
+
if value is not None:
|
|
62
|
+
values[key] = value
|
|
63
|
+
|
|
64
|
+
for key, value in os.environ.items():
|
|
65
|
+
values[key] = value
|
|
66
|
+
return values
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _load_credentials(config_path: Path, env: Mapping[str, str]) -> dict[str, Credential]:
|
|
70
|
+
data = _read_toml(config_path)
|
|
71
|
+
raw_credentials = data.get("credentials", {})
|
|
72
|
+
if not isinstance(raw_credentials, dict):
|
|
73
|
+
raise ConfigError("sci/config.toml field credentials must be a table")
|
|
74
|
+
|
|
75
|
+
credentials: dict[str, Credential] = {}
|
|
76
|
+
for credential_id, raw in raw_credentials.items():
|
|
77
|
+
section = _require_mapping(raw, f"credentials.{credential_id}")
|
|
78
|
+
type_ = _require_str(section, "type", f"credentials.{credential_id}")
|
|
79
|
+
username_env = _require_str(section, "username_env", f"credentials.{credential_id}")
|
|
80
|
+
password_env = _require_str(section, "password_env", f"credentials.{credential_id}")
|
|
81
|
+
username = _require_env(env, username_env)
|
|
82
|
+
password = _require_env(env, password_env)
|
|
83
|
+
credentials[credential_id] = Credential(
|
|
84
|
+
id=credential_id,
|
|
85
|
+
type=type_,
|
|
86
|
+
username_env=username_env,
|
|
87
|
+
password_env=password_env,
|
|
88
|
+
username=username,
|
|
89
|
+
password=password,
|
|
90
|
+
)
|
|
91
|
+
return credentials
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _load_targets(targets_dir: Path, env: Mapping[str, str]) -> dict[str, Target]:
|
|
95
|
+
targets: dict[str, Target] = {}
|
|
96
|
+
if not targets_dir.exists():
|
|
97
|
+
return targets
|
|
98
|
+
|
|
99
|
+
for config_path in sorted(targets_dir.glob("*.toml")):
|
|
100
|
+
target_id = config_path.stem
|
|
101
|
+
data = _read_toml(config_path)
|
|
102
|
+
name = _require_str(data, "name", f"target {target_id}")
|
|
103
|
+
server_section = data.get("server", {})
|
|
104
|
+
server = _build_target_server(target_id, _require_mapping(server_section, f"target {target_id}.server"), env)
|
|
105
|
+
targets[target_id] = Target(
|
|
106
|
+
id=target_id,
|
|
107
|
+
name=name,
|
|
108
|
+
server=server,
|
|
109
|
+
vars=_optional_mapping(data, "vars", f"target {target_id}"),
|
|
110
|
+
description=_optional_str(data, "description"),
|
|
111
|
+
policy=_optional_mapping(data, "policy", f"target {target_id}"),
|
|
112
|
+
)
|
|
113
|
+
return targets
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _build_target_server(target_id: str, server: Mapping[str, Any], env: Mapping[str, str]) -> TargetServer:
|
|
117
|
+
normalized_id = _normalize_env_id(target_id)
|
|
118
|
+
url_env = _optional_str(server, "url_env") or f"SERVER_{normalized_id}_URL"
|
|
119
|
+
token_env = _optional_str(server, "token_env") or f"SERVER_{normalized_id}_TOKEN"
|
|
120
|
+
return TargetServer(url=_require_env(env, url_env), token=_require_env(env, token_env))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _load_projects(
|
|
124
|
+
projects_dir: Path,
|
|
125
|
+
credentials: Mapping[str, Credential],
|
|
126
|
+
) -> tuple[dict[str, Project], dict[str, Pipeline]]:
|
|
127
|
+
projects: dict[str, Project] = {}
|
|
128
|
+
pipelines: dict[str, Pipeline] = {}
|
|
129
|
+
if not projects_dir.exists():
|
|
130
|
+
return projects, pipelines
|
|
131
|
+
|
|
132
|
+
for project_dir in sorted(path for path in projects_dir.iterdir() if path.is_dir()):
|
|
133
|
+
project_id = project_dir.name
|
|
134
|
+
project = _load_project(project_id, project_dir / "config.toml", credentials)
|
|
135
|
+
projects[project_id] = project
|
|
136
|
+
pipelines.update(_load_pipelines(project_id, project_dir / "pipeline"))
|
|
137
|
+
return projects, pipelines
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _load_project(project_id: str, config_path: Path, credentials: Mapping[str, Credential]) -> Project:
|
|
141
|
+
data = _read_toml(config_path)
|
|
142
|
+
name = _require_str(data, "name", f"project {project_id}")
|
|
143
|
+
source_section = _require_mapping(data.get("source"), f"project {project_id}.source")
|
|
144
|
+
credential = _optional_str(source_section, "credential") or None
|
|
145
|
+
if credential and credential not in credentials:
|
|
146
|
+
raise ConfigError(f"project {project_id} references unknown credential: {credential}")
|
|
147
|
+
|
|
148
|
+
return Project(
|
|
149
|
+
id=project_id,
|
|
150
|
+
name=name,
|
|
151
|
+
description=_optional_str(data, "description"),
|
|
152
|
+
vars=_optional_mapping(data, "vars", f"project {project_id}"),
|
|
153
|
+
source=ProjectSource(
|
|
154
|
+
repo=_require_str(source_section, "repo", f"project {project_id}.source"),
|
|
155
|
+
branch=_require_str(source_section, "branch", f"project {project_id}.source"),
|
|
156
|
+
credential=credential,
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _load_pipelines(project_id: str, pipeline_root: Path) -> dict[str, Pipeline]:
|
|
162
|
+
pipelines: dict[str, Pipeline] = {}
|
|
163
|
+
if not pipeline_root.exists():
|
|
164
|
+
return pipelines
|
|
165
|
+
|
|
166
|
+
for pipeline_dir in sorted(path for path in pipeline_root.iterdir() if path.is_dir()):
|
|
167
|
+
pipeline_id = pipeline_dir.name
|
|
168
|
+
pipelines[f"{project_id}/{pipeline_id}"] = _load_pipeline(pipeline_id, pipeline_dir)
|
|
169
|
+
return pipelines
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _load_pipeline(pipeline_id: str, pipeline_dir: Path) -> Pipeline:
|
|
173
|
+
config = _read_toml(pipeline_dir / "config.toml")
|
|
174
|
+
pipeline_yml = _read_yaml_mapping(pipeline_dir / "pipeline.yml")
|
|
175
|
+
steps = pipeline_yml.get("steps")
|
|
176
|
+
if not isinstance(steps, list):
|
|
177
|
+
raise ConfigError(f"pipeline {pipeline_id} field steps must be a list")
|
|
178
|
+
|
|
179
|
+
return Pipeline(
|
|
180
|
+
id=pipeline_id,
|
|
181
|
+
name=_require_str(config, "name", f"pipeline {pipeline_id}"),
|
|
182
|
+
description=_optional_str(config, "description"),
|
|
183
|
+
vars=_optional_mapping(config, "vars", f"pipeline {pipeline_id}"),
|
|
184
|
+
steps=[_load_step(index, raw_step, pipeline_id) for index, raw_step in enumerate(steps)],
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _load_step(index: int, raw_step: Any, pipeline_id: str) -> Step:
|
|
189
|
+
context = f"pipeline {pipeline_id} step {index + 1}"
|
|
190
|
+
data = _require_mapping(raw_step, context)
|
|
191
|
+
return Step(
|
|
192
|
+
name=_require_str(data, "name", context),
|
|
193
|
+
uses=_require_str(data, "uses", context),
|
|
194
|
+
with_=_optional_mapping(data, "with", context),
|
|
195
|
+
when=_optional_str(data, "when") or None,
|
|
196
|
+
pre=_optional_list(data, "pre", context),
|
|
197
|
+
after=_optional_list(data, "after", context),
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _read_toml(path: Path) -> dict[str, Any]:
|
|
202
|
+
if not path.exists():
|
|
203
|
+
raise ConfigError(f"missing config file: {path}")
|
|
204
|
+
try:
|
|
205
|
+
with path.open("rb") as file_obj:
|
|
206
|
+
data = tomllib.load(file_obj)
|
|
207
|
+
except tomllib.TOMLDecodeError as exc:
|
|
208
|
+
raise ConfigError(f"invalid TOML file {path}: {exc}") from exc
|
|
209
|
+
if not isinstance(data, dict):
|
|
210
|
+
raise ConfigError(f"TOML file must contain a table: {path}")
|
|
211
|
+
return data
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _read_yaml_mapping(path: Path) -> dict[str, Any]:
|
|
215
|
+
if not path.exists():
|
|
216
|
+
raise ConfigError(f"missing pipeline file: {path}")
|
|
217
|
+
try:
|
|
218
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
219
|
+
except yaml.YAMLError as exc:
|
|
220
|
+
raise ConfigError(f"invalid YAML file {path}: {exc}") from exc
|
|
221
|
+
if data is None:
|
|
222
|
+
data = {}
|
|
223
|
+
return dict(_require_mapping(data, f"pipeline file {path}"))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _require_mapping(value: Any, context: str) -> Mapping[str, Any]:
|
|
227
|
+
if not isinstance(value, dict):
|
|
228
|
+
raise ConfigError(f"{context} must be a table")
|
|
229
|
+
return value
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _require_str(data: Mapping[str, Any], field: str, context: str) -> str:
|
|
233
|
+
value = data.get(field)
|
|
234
|
+
if not isinstance(value, str) or not value:
|
|
235
|
+
raise ConfigError(f"{context} missing required field: {field}")
|
|
236
|
+
return value
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _optional_str(data: Mapping[str, Any], field: str) -> str:
|
|
240
|
+
value = data.get(field, "")
|
|
241
|
+
if value is None:
|
|
242
|
+
return ""
|
|
243
|
+
if not isinstance(value, str):
|
|
244
|
+
raise ConfigError(f"field {field} must be a string")
|
|
245
|
+
return value
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _optional_mapping(data: Mapping[str, Any], field: str, context: str) -> dict[str, Any]:
|
|
249
|
+
value = data.get(field, {})
|
|
250
|
+
if not isinstance(value, dict):
|
|
251
|
+
raise ConfigError(f"{context} field {field} must be a table")
|
|
252
|
+
return dict(value)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _optional_list(data: Mapping[str, Any], field: str, context: str) -> list[dict[str, Any]]:
|
|
256
|
+
value = data.get(field, [])
|
|
257
|
+
if not isinstance(value, list):
|
|
258
|
+
raise ConfigError(f"{context} field {field} must be a list")
|
|
259
|
+
for item in value:
|
|
260
|
+
if not isinstance(item, dict):
|
|
261
|
+
raise ConfigError(f"{context} field {field} items must be tables")
|
|
262
|
+
return [dict(item) for item in value]
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _require_env(env: Mapping[str, str], env_name: str) -> str:
|
|
266
|
+
value = env.get(env_name)
|
|
267
|
+
if value is None or value == "":
|
|
268
|
+
raise ConfigError(f"missing required environment variable: {env_name}")
|
|
269
|
+
return value
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _normalize_env_id(value: str) -> str:
|
|
273
|
+
normalized = re.sub(r"[^0-9A-Za-z]+", "_", value).strip("_").upper()
|
|
274
|
+
if not normalized:
|
|
275
|
+
raise ConfigError("target id cannot be empty after normalization")
|
|
276
|
+
return normalized
|