ls-algorithm-plugin-sdk 0.3.6__tar.gz → 0.3.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/PKG-INFO +10 -2
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/README.md +10 -2
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/pyproject.toml +1 -1
- ls_algorithm_plugin_sdk-0.3.7/src/algorithm_plugin_sdk/cleanup.py +336 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli_impl/configure.py +33 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli_impl/serve.py +22 -8
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/deployment.py +20 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/service.py +8 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/ls_algorithm_plugin_sdk.egg-info/PKG-INFO +10 -2
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/ls_algorithm_plugin_sdk.egg-info/SOURCES.txt +2 -0
- ls_algorithm_plugin_sdk-0.3.7/tests/test_cleanup.py +98 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_configure.py +9 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_deployment.py +22 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_registration.py +5 -1
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_sdk_automation.py +4 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_serve.py +6 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_service.py +10 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/setup.cfg +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/__init__.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/algorithm.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli_impl/__init__.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli_impl/cut.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli_impl/parsing.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli_impl/run.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/context.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/errors.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/examples/__init__.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/examples/example_algorithm.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/examples/simulated_algorithm.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/gpu_isolation.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/loader.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/log_manager.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/models.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/registration.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/release.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/runner.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/webui/__init__.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/webui/app.css +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/webui/app.js +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/webui/index.html +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/webui_app.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/ls_algorithm_plugin_sdk.egg-info/dependency_links.txt +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/ls_algorithm_plugin_sdk.egg-info/entry_points.txt +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/ls_algorithm_plugin_sdk.egg-info/requires.txt +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/ls_algorithm_plugin_sdk.egg-info/top_level.txt +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_cli.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_cut.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_gpu_isolation.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_models.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_release.py +0 -0
- {ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_runner.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ls-algorithm-plugin-sdk
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.7
|
|
4
4
|
Summary: Protocol-independent runtime SDK for dataset algorithms
|
|
5
5
|
Author: Ling Robotics
|
|
6
6
|
License: Proprietary
|
|
@@ -71,11 +71,19 @@ sudo /srv/camera-space-mano/.venv/bin/algorithm-plugin configure \
|
|
|
71
71
|
--api-key "$LDP_INTERNAL_API_KEY" \
|
|
72
72
|
--port 9030 \
|
|
73
73
|
--cluster production \
|
|
74
|
-
--gpu-ids 0,1
|
|
74
|
+
--gpu-ids 0,1 \
|
|
75
|
+
--log-retention-days 7 \
|
|
76
|
+
--output-retention-days 2 \
|
|
77
|
+
--scratch-retention-days 2
|
|
75
78
|
```
|
|
76
79
|
|
|
77
80
|
重复执行 configure 需要显式提供 `--force`。
|
|
78
81
|
|
|
82
|
+
配置部署的服务会启动独立清理进程,每小时扫描一次。仅删除创建超过日志保留期且
|
|
83
|
+
末行状态为 `succeeded` 的任务日志;`outputRoot` 和 `scratchRoot` 下超过对应保留期
|
|
84
|
+
未修改的数据也会被删除。清理记录按天写入 `<repository>/logs/cleanup-YYYYMMDD.log`,
|
|
85
|
+
包含清理时间、路径和字节数;清理记录自身不会被自动删除。
|
|
86
|
+
|
|
79
87
|
确认输出后执行:
|
|
80
88
|
|
|
81
89
|
```bash
|
|
@@ -55,11 +55,19 @@ sudo /srv/camera-space-mano/.venv/bin/algorithm-plugin configure \
|
|
|
55
55
|
--api-key "$LDP_INTERNAL_API_KEY" \
|
|
56
56
|
--port 9030 \
|
|
57
57
|
--cluster production \
|
|
58
|
-
--gpu-ids 0,1
|
|
58
|
+
--gpu-ids 0,1 \
|
|
59
|
+
--log-retention-days 7 \
|
|
60
|
+
--output-retention-days 2 \
|
|
61
|
+
--scratch-retention-days 2
|
|
59
62
|
```
|
|
60
63
|
|
|
61
64
|
重复执行 configure 需要显式提供 `--force`。
|
|
62
65
|
|
|
66
|
+
配置部署的服务会启动独立清理进程,每小时扫描一次。仅删除创建超过日志保留期且
|
|
67
|
+
末行状态为 `succeeded` 的任务日志;`outputRoot` 和 `scratchRoot` 下超过对应保留期
|
|
68
|
+
未修改的数据也会被删除。清理记录按天写入 `<repository>/logs/cleanup-YYYYMMDD.log`,
|
|
69
|
+
包含清理时间、路径和字节数;清理记录自身不会被自动删除。
|
|
70
|
+
|
|
63
71
|
确认输出后执行:
|
|
64
72
|
|
|
65
73
|
```bash
|
|
@@ -81,4 +89,4 @@ sudo systemctl status camera-space-mano.service --no-pager
|
|
|
81
89
|
```bash
|
|
82
90
|
python -m pip install .
|
|
83
91
|
python -m unittest discover -s tests -v
|
|
84
|
-
```
|
|
92
|
+
```
|
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import multiprocessing
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import time
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
DEFAULT_LOG_RETENTION_DAYS = 7
|
|
13
|
+
DEFAULT_OUTPUT_RETENTION_DAYS = 2
|
|
14
|
+
DEFAULT_SCRATCH_RETENTION_DAYS = 2
|
|
15
|
+
SCAN_INTERVAL_SECONDS = 60 * 60
|
|
16
|
+
|
|
17
|
+
_AUDIT_LOG_PREFIX = "cleanup-"
|
|
18
|
+
_SECONDS_PER_DAY = 24 * 60 * 60
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _write_audit(log_dir: Path, event: str, **values: Any) -> None:
|
|
22
|
+
log_dir.mkdir(parents=True, exist_ok=True)
|
|
23
|
+
timestamp = datetime.now().astimezone()
|
|
24
|
+
path = log_dir / f"{_AUDIT_LOG_PREFIX}{timestamp:%Y%m%d}.log"
|
|
25
|
+
payload = {
|
|
26
|
+
"timestamp": timestamp.isoformat(),
|
|
27
|
+
"event": event,
|
|
28
|
+
**values,
|
|
29
|
+
}
|
|
30
|
+
with path.open("a", encoding="utf-8", buffering=1) as stream:
|
|
31
|
+
stream.write(json.dumps(payload, ensure_ascii=False, default=str) + "\n")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _job_logs(log_dir: Path) -> list[Path]:
|
|
35
|
+
return sorted(
|
|
36
|
+
path
|
|
37
|
+
for path in log_dir.glob("*.log")
|
|
38
|
+
if not path.name.startswith(_AUDIT_LOG_PREFIX) and path.is_file()
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _request_roots(path: Path) -> tuple[set[Path], set[Path]]:
|
|
43
|
+
output_roots: set[Path] = set()
|
|
44
|
+
scratch_roots: set[Path] = set()
|
|
45
|
+
with path.open(encoding="utf-8", errors="replace") as stream:
|
|
46
|
+
for line in stream:
|
|
47
|
+
try:
|
|
48
|
+
payload = json.loads(line)
|
|
49
|
+
except json.JSONDecodeError:
|
|
50
|
+
continue
|
|
51
|
+
if payload.get("event") != "request":
|
|
52
|
+
continue
|
|
53
|
+
output_root = _safe_root(payload.get("outputRoot"))
|
|
54
|
+
scratch_root = _safe_root(payload.get("scratchRoot"))
|
|
55
|
+
if output_root is not None:
|
|
56
|
+
output_roots.add(output_root)
|
|
57
|
+
if scratch_root is not None:
|
|
58
|
+
scratch_roots.add(scratch_root)
|
|
59
|
+
return output_roots, scratch_roots
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _safe_root(value: Any) -> Path | None:
|
|
63
|
+
if not isinstance(value, str) or not value.strip():
|
|
64
|
+
return None
|
|
65
|
+
path = Path(value).expanduser()
|
|
66
|
+
if not path.is_absolute():
|
|
67
|
+
return None
|
|
68
|
+
resolved = path.resolve()
|
|
69
|
+
if resolved == Path(resolved.anchor):
|
|
70
|
+
return None
|
|
71
|
+
return resolved
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _last_nonempty_line(path: Path) -> str:
|
|
75
|
+
with path.open("rb") as stream:
|
|
76
|
+
stream.seek(0, os.SEEK_END)
|
|
77
|
+
position = stream.tell()
|
|
78
|
+
data = b""
|
|
79
|
+
while position > 0:
|
|
80
|
+
read_size = min(8192, position)
|
|
81
|
+
position -= read_size
|
|
82
|
+
stream.seek(position)
|
|
83
|
+
data = stream.read(read_size) + data
|
|
84
|
+
lines = data.splitlines()
|
|
85
|
+
if position == 0 or len(lines) > 1:
|
|
86
|
+
for line in reversed(lines):
|
|
87
|
+
if line.strip():
|
|
88
|
+
return line.decode("utf-8", errors="replace")
|
|
89
|
+
return ""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _succeeded(path: Path) -> bool:
|
|
93
|
+
try:
|
|
94
|
+
payload = json.loads(_last_nonempty_line(path))
|
|
95
|
+
except (OSError, json.JSONDecodeError):
|
|
96
|
+
return False
|
|
97
|
+
return payload.get("state") == "succeeded"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _log_created_at(path: Path) -> float:
|
|
101
|
+
timestamp, separator, _ = path.name.partition("-")
|
|
102
|
+
if separator and len(timestamp) == 14 and timestamp.isdigit():
|
|
103
|
+
try:
|
|
104
|
+
return datetime.strptime(timestamp, "%Y%m%d%H%M%S").replace(
|
|
105
|
+
tzinfo=timezone.utc
|
|
106
|
+
).timestamp()
|
|
107
|
+
except ValueError:
|
|
108
|
+
pass
|
|
109
|
+
stat = path.stat()
|
|
110
|
+
return getattr(stat, "st_birthtime", stat.st_ctime)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _tree_stats(path: Path) -> tuple[int, float]:
|
|
114
|
+
stat = path.lstat()
|
|
115
|
+
total_size = stat.st_size if not path.is_dir() or path.is_symlink() else 0
|
|
116
|
+
latest_mtime = stat.st_mtime
|
|
117
|
+
if path.is_dir() and not path.is_symlink():
|
|
118
|
+
for root, directories, files in os.walk(path, followlinks=False):
|
|
119
|
+
root_path = Path(root)
|
|
120
|
+
for name in (*directories, *files):
|
|
121
|
+
child = root_path / name
|
|
122
|
+
try:
|
|
123
|
+
child_stat = child.lstat()
|
|
124
|
+
except FileNotFoundError:
|
|
125
|
+
continue
|
|
126
|
+
latest_mtime = max(latest_mtime, child_stat.st_mtime)
|
|
127
|
+
if not child.is_dir() or child.is_symlink():
|
|
128
|
+
total_size += child_stat.st_size
|
|
129
|
+
return total_size, latest_mtime
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _delete(path: Path) -> None:
|
|
133
|
+
if path.is_dir() and not path.is_symlink():
|
|
134
|
+
shutil.rmtree(path)
|
|
135
|
+
else:
|
|
136
|
+
path.unlink()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def cleanup_once(
|
|
140
|
+
log_dir: str | Path,
|
|
141
|
+
*,
|
|
142
|
+
log_retention_days: int = DEFAULT_LOG_RETENTION_DAYS,
|
|
143
|
+
output_retention_days: int = DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
144
|
+
scratch_retention_days: int = DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
145
|
+
known_output_roots: set[Path] | None = None,
|
|
146
|
+
known_scratch_roots: set[Path] | None = None,
|
|
147
|
+
now: float | None = None,
|
|
148
|
+
) -> tuple[set[Path], set[Path]]:
|
|
149
|
+
directory = Path(log_dir).expanduser().resolve()
|
|
150
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
151
|
+
output_roots = known_output_roots if known_output_roots is not None else set()
|
|
152
|
+
scratch_roots = known_scratch_roots if known_scratch_roots is not None else set()
|
|
153
|
+
current_time = time.time() if now is None else now
|
|
154
|
+
started_at = datetime.now().astimezone().isoformat()
|
|
155
|
+
deleted_count = 0
|
|
156
|
+
deleted_bytes = 0
|
|
157
|
+
error_count = 0
|
|
158
|
+
|
|
159
|
+
logs = _job_logs(directory)
|
|
160
|
+
for path in logs:
|
|
161
|
+
try:
|
|
162
|
+
discovered_output, discovered_scratch = _request_roots(path)
|
|
163
|
+
except OSError as exc:
|
|
164
|
+
error_count += 1
|
|
165
|
+
_write_audit(
|
|
166
|
+
directory,
|
|
167
|
+
"scan_failed",
|
|
168
|
+
category="log",
|
|
169
|
+
path=str(path),
|
|
170
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
171
|
+
)
|
|
172
|
+
continue
|
|
173
|
+
output_roots.update(discovered_output)
|
|
174
|
+
scratch_roots.update(discovered_scratch)
|
|
175
|
+
|
|
176
|
+
def remove(path: Path, category: str, size: int) -> None:
|
|
177
|
+
nonlocal deleted_count, deleted_bytes, error_count
|
|
178
|
+
try:
|
|
179
|
+
_delete(path)
|
|
180
|
+
except FileNotFoundError:
|
|
181
|
+
return
|
|
182
|
+
except OSError as exc:
|
|
183
|
+
error_count += 1
|
|
184
|
+
_write_audit(
|
|
185
|
+
directory,
|
|
186
|
+
"delete_failed",
|
|
187
|
+
category=category,
|
|
188
|
+
path=str(path),
|
|
189
|
+
sizeBytes=size,
|
|
190
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
191
|
+
)
|
|
192
|
+
return
|
|
193
|
+
deleted_count += 1
|
|
194
|
+
deleted_bytes += size
|
|
195
|
+
_write_audit(
|
|
196
|
+
directory,
|
|
197
|
+
"deleted",
|
|
198
|
+
category=category,
|
|
199
|
+
path=str(path),
|
|
200
|
+
sizeBytes=size,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
log_cutoff = current_time - log_retention_days * _SECONDS_PER_DAY
|
|
204
|
+
for path in logs:
|
|
205
|
+
try:
|
|
206
|
+
if _log_created_at(path) < log_cutoff and _succeeded(path):
|
|
207
|
+
remove(path, "log", path.stat().st_size)
|
|
208
|
+
except OSError as exc:
|
|
209
|
+
error_count += 1
|
|
210
|
+
_write_audit(
|
|
211
|
+
directory,
|
|
212
|
+
"scan_failed",
|
|
213
|
+
category="log",
|
|
214
|
+
path=str(path),
|
|
215
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
protected = directory.resolve()
|
|
219
|
+
for category, roots, retention_days in (
|
|
220
|
+
("output", output_roots, output_retention_days),
|
|
221
|
+
("scratch", scratch_roots, scratch_retention_days),
|
|
222
|
+
):
|
|
223
|
+
cutoff = current_time - retention_days * _SECONDS_PER_DAY
|
|
224
|
+
for root in sorted(roots):
|
|
225
|
+
if not root.is_dir():
|
|
226
|
+
continue
|
|
227
|
+
try:
|
|
228
|
+
entries = list(root.iterdir())
|
|
229
|
+
except OSError as exc:
|
|
230
|
+
error_count += 1
|
|
231
|
+
_write_audit(
|
|
232
|
+
directory,
|
|
233
|
+
"scan_failed",
|
|
234
|
+
category=category,
|
|
235
|
+
path=str(root),
|
|
236
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
237
|
+
)
|
|
238
|
+
continue
|
|
239
|
+
for path in entries:
|
|
240
|
+
resolved = path.resolve()
|
|
241
|
+
if (
|
|
242
|
+
resolved == protected
|
|
243
|
+
or resolved in protected.parents
|
|
244
|
+
or protected in resolved.parents
|
|
245
|
+
):
|
|
246
|
+
continue
|
|
247
|
+
try:
|
|
248
|
+
size, latest_mtime = _tree_stats(path)
|
|
249
|
+
if latest_mtime < cutoff:
|
|
250
|
+
remove(path, category, size)
|
|
251
|
+
except OSError as exc:
|
|
252
|
+
error_count += 1
|
|
253
|
+
_write_audit(
|
|
254
|
+
directory,
|
|
255
|
+
"scan_failed",
|
|
256
|
+
category=category,
|
|
257
|
+
path=str(path),
|
|
258
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
_write_audit(
|
|
262
|
+
directory,
|
|
263
|
+
"scan_completed",
|
|
264
|
+
startedAt=started_at,
|
|
265
|
+
finishedAt=datetime.now().astimezone().isoformat(),
|
|
266
|
+
deletedCount=deleted_count,
|
|
267
|
+
deletedBytes=deleted_bytes,
|
|
268
|
+
errorCount=error_count,
|
|
269
|
+
)
|
|
270
|
+
return output_roots, scratch_roots
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _cleanup_loop(
|
|
274
|
+
log_dir: str,
|
|
275
|
+
log_retention_days: int,
|
|
276
|
+
output_retention_days: int,
|
|
277
|
+
scratch_retention_days: int,
|
|
278
|
+
stop_event: Any,
|
|
279
|
+
) -> None:
|
|
280
|
+
output_roots: set[Path] = set()
|
|
281
|
+
scratch_roots: set[Path] = set()
|
|
282
|
+
while True:
|
|
283
|
+
try:
|
|
284
|
+
cleanup_once(
|
|
285
|
+
log_dir,
|
|
286
|
+
log_retention_days=log_retention_days,
|
|
287
|
+
output_retention_days=output_retention_days,
|
|
288
|
+
scratch_retention_days=scratch_retention_days,
|
|
289
|
+
known_output_roots=output_roots,
|
|
290
|
+
known_scratch_roots=scratch_roots,
|
|
291
|
+
)
|
|
292
|
+
except Exception as exc:
|
|
293
|
+
_write_audit(
|
|
294
|
+
Path(log_dir),
|
|
295
|
+
"scan_crashed",
|
|
296
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
297
|
+
)
|
|
298
|
+
if stop_event.wait(SCAN_INTERVAL_SECONDS):
|
|
299
|
+
return
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class CleanupProcess:
|
|
303
|
+
def __init__(
|
|
304
|
+
self,
|
|
305
|
+
log_dir: str | Path,
|
|
306
|
+
*,
|
|
307
|
+
log_retention_days: int = DEFAULT_LOG_RETENTION_DAYS,
|
|
308
|
+
output_retention_days: int = DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
309
|
+
scratch_retention_days: int = DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
310
|
+
) -> None:
|
|
311
|
+
context = multiprocessing.get_context("spawn")
|
|
312
|
+
self._stop_event = context.Event()
|
|
313
|
+
self._process = context.Process(
|
|
314
|
+
target=_cleanup_loop,
|
|
315
|
+
args=(
|
|
316
|
+
str(Path(log_dir).expanduser().resolve()),
|
|
317
|
+
log_retention_days,
|
|
318
|
+
output_retention_days,
|
|
319
|
+
scratch_retention_days,
|
|
320
|
+
self._stop_event,
|
|
321
|
+
),
|
|
322
|
+
name="algorithm-plugin-cleanup",
|
|
323
|
+
daemon=True,
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
def start(self) -> None:
|
|
327
|
+
self._process.start()
|
|
328
|
+
|
|
329
|
+
def stop(self) -> None:
|
|
330
|
+
if not self._process.is_alive():
|
|
331
|
+
return
|
|
332
|
+
self._stop_event.set()
|
|
333
|
+
self._process.join(timeout=5)
|
|
334
|
+
if self._process.is_alive():
|
|
335
|
+
self._process.terminate()
|
|
336
|
+
self._process.join(timeout=5)
|
|
@@ -10,6 +10,11 @@ import sys
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
from urllib.parse import urlparse
|
|
12
12
|
|
|
13
|
+
from ..cleanup import (
|
|
14
|
+
DEFAULT_LOG_RETENTION_DAYS,
|
|
15
|
+
DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
16
|
+
DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
17
|
+
)
|
|
13
18
|
from ..deployment import (
|
|
14
19
|
DEFAULT_CONFIG_NAME,
|
|
15
20
|
DeploymentConfig,
|
|
@@ -31,6 +36,13 @@ SERVICE_NAME_PATTERN = re.compile(
|
|
|
31
36
|
)
|
|
32
37
|
|
|
33
38
|
|
|
39
|
+
def _retention_days(value: str) -> int:
|
|
40
|
+
days = int(value)
|
|
41
|
+
if days < 0:
|
|
42
|
+
raise argparse.ArgumentTypeError("retention days must be non-negative")
|
|
43
|
+
return days
|
|
44
|
+
|
|
45
|
+
|
|
34
46
|
def configure_parser(parser: argparse.ArgumentParser) -> None:
|
|
35
47
|
parser.add_argument(
|
|
36
48
|
"algorithm",
|
|
@@ -72,6 +84,24 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
|
|
|
72
84
|
action="store_true",
|
|
73
85
|
help="override request output paths with <repository>/.output/<job-id>",
|
|
74
86
|
)
|
|
87
|
+
parser.add_argument(
|
|
88
|
+
"--log-retention-days",
|
|
89
|
+
type=_retention_days,
|
|
90
|
+
default=DEFAULT_LOG_RETENTION_DAYS,
|
|
91
|
+
help=f"successful log retention in days (default: {DEFAULT_LOG_RETENTION_DAYS})",
|
|
92
|
+
)
|
|
93
|
+
parser.add_argument(
|
|
94
|
+
"--output-retention-days",
|
|
95
|
+
type=_retention_days,
|
|
96
|
+
default=DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
97
|
+
help=f"output retention in days (default: {DEFAULT_OUTPUT_RETENTION_DAYS})",
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument(
|
|
100
|
+
"--scratch-retention-days",
|
|
101
|
+
type=_retention_days,
|
|
102
|
+
default=DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
103
|
+
help=f"scratch retention in days (default: {DEFAULT_SCRATCH_RETENTION_DAYS})",
|
|
104
|
+
)
|
|
75
105
|
parser.add_argument(
|
|
76
106
|
"--gpu-ids",
|
|
77
107
|
type=gpu_ids,
|
|
@@ -426,6 +456,9 @@ def generate_config(
|
|
|
426
456
|
gpu_ids=args.gpu_ids,
|
|
427
457
|
environment=dict(args.env),
|
|
428
458
|
default_parameters=args.default_parameters,
|
|
459
|
+
log_retention_days=args.log_retention_days,
|
|
460
|
+
output_retention_days=args.output_retention_days,
|
|
461
|
+
scratch_retention_days=args.scratch_retention_days,
|
|
429
462
|
)
|
|
430
463
|
config.save()
|
|
431
464
|
return config
|
|
@@ -4,6 +4,7 @@ import argparse
|
|
|
4
4
|
import os
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
|
|
7
|
+
from ..cleanup import CleanupProcess
|
|
7
8
|
from ..deployment import DEFAULT_CONFIG_NAME, DeploymentConfig
|
|
8
9
|
from ..gpu_isolation import apply_gpu_isolation
|
|
9
10
|
from ..loader import load_algorithm
|
|
@@ -139,12 +140,25 @@ def execute(args: argparse.Namespace) -> int:
|
|
|
139
140
|
raise RuntimeError(
|
|
140
141
|
"compute registration requires the generated release-manifest.json; run configure first"
|
|
141
142
|
)
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
143
|
+
cleanup = None
|
|
144
|
+
if config is not None and service is not None:
|
|
145
|
+
cleanup = CleanupProcess(
|
|
146
|
+
config.repository / "logs",
|
|
147
|
+
log_retention_days=int(service["logRetentionDays"]),
|
|
148
|
+
output_retention_days=int(service["outputRetentionDays"]),
|
|
149
|
+
scratch_retention_days=int(service["scratchRetentionDays"]),
|
|
150
|
+
)
|
|
151
|
+
cleanup.start()
|
|
152
|
+
try:
|
|
153
|
+
run_server(
|
|
154
|
+
manager,
|
|
155
|
+
host=args.host,
|
|
156
|
+
port=args.port,
|
|
157
|
+
webui=args.webui,
|
|
158
|
+
token=os.getenv("LDP_PLUGIN_TOKEN") or None,
|
|
159
|
+
lifecycle=registration,
|
|
160
|
+
)
|
|
161
|
+
finally:
|
|
162
|
+
if cleanup is not None:
|
|
163
|
+
cleanup.stop()
|
|
150
164
|
return 0
|
|
@@ -11,6 +11,11 @@ from pathlib import Path
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
from urllib.parse import urlparse
|
|
13
13
|
|
|
14
|
+
from .cleanup import (
|
|
15
|
+
DEFAULT_LOG_RETENTION_DAYS,
|
|
16
|
+
DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
17
|
+
DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
18
|
+
)
|
|
14
19
|
from .loader import ALGORITHM_ENTRY_POINT_GROUP, load_algorithm
|
|
15
20
|
from .release import ReleaseManifest
|
|
16
21
|
|
|
@@ -148,6 +153,9 @@ class DeploymentConfig:
|
|
|
148
153
|
default_parameters: dict[str, Any] | None = None,
|
|
149
154
|
local_scratch: bool = False,
|
|
150
155
|
local_output: bool = False,
|
|
156
|
+
log_retention_days: int = DEFAULT_LOG_RETENTION_DAYS,
|
|
157
|
+
output_retention_days: int = DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
158
|
+
scratch_retention_days: int = DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
151
159
|
) -> "DeploymentConfig":
|
|
152
160
|
repository = repository_root(root)
|
|
153
161
|
_make_repository_importable(repository)
|
|
@@ -196,6 +204,9 @@ class DeploymentConfig:
|
|
|
196
204
|
"gpuIds": configured_gpu_ids,
|
|
197
205
|
"token": token or None,
|
|
198
206
|
"defaultParameters": dict(default_parameters or {}),
|
|
207
|
+
"logRetentionDays": log_retention_days,
|
|
208
|
+
"outputRetentionDays": output_retention_days,
|
|
209
|
+
"scratchRetentionDays": scratch_retention_days,
|
|
199
210
|
},
|
|
200
211
|
"registration": registration,
|
|
201
212
|
"environment": dict(sorted((environment or {}).items())),
|
|
@@ -275,6 +286,15 @@ class DeploymentConfig:
|
|
|
275
286
|
for name in ("localScratch", "localOutput"):
|
|
276
287
|
if not isinstance(service.get(name, False), bool):
|
|
277
288
|
raise RuntimeError(f"service {name} must be a boolean")
|
|
289
|
+
retention_defaults = {
|
|
290
|
+
"logRetentionDays": DEFAULT_LOG_RETENTION_DAYS,
|
|
291
|
+
"outputRetentionDays": DEFAULT_OUTPUT_RETENTION_DAYS,
|
|
292
|
+
"scratchRetentionDays": DEFAULT_SCRATCH_RETENTION_DAYS,
|
|
293
|
+
}
|
|
294
|
+
for name, default in retention_defaults.items():
|
|
295
|
+
value = service.setdefault(name, default)
|
|
296
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
297
|
+
raise RuntimeError(f"service {name} must be a non-negative integer")
|
|
278
298
|
gpu_ids = service.get("gpuIds")
|
|
279
299
|
if not isinstance(gpu_ids, list) or any(
|
|
280
300
|
isinstance(gpu_id, bool) or not isinstance(gpu_id, int) or gpu_id < 0
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/service.py
RENAMED
|
@@ -348,6 +348,11 @@ class ExecutionManager:
|
|
|
348
348
|
"acceptedAt": record.accepted_at,
|
|
349
349
|
}
|
|
350
350
|
|
|
351
|
+
def _resources(self) -> dict[str, Any]:
|
|
352
|
+
if not self.gpu_ids:
|
|
353
|
+
return {}
|
|
354
|
+
return {"gpuIds": list(self.gpu_ids)}
|
|
355
|
+
|
|
351
356
|
def manifest(self) -> dict[str, Any]:
|
|
352
357
|
metadata = self.runner.algorithm.metadata()
|
|
353
358
|
capabilities = {
|
|
@@ -361,6 +366,7 @@ class ExecutionManager:
|
|
|
361
366
|
manifest = self.release_manifest.plugin_manifest(
|
|
362
367
|
max_concurrency=self.max_concurrent_executions,
|
|
363
368
|
capabilities=capabilities,
|
|
369
|
+
resources=self._resources(),
|
|
364
370
|
)
|
|
365
371
|
manifest["metadata"] = metadata.to_dict()
|
|
366
372
|
return manifest
|
|
@@ -374,6 +380,7 @@ class ExecutionManager:
|
|
|
374
380
|
"maxConcurrency": self.max_concurrent_executions,
|
|
375
381
|
"capabilities": capabilities,
|
|
376
382
|
"metadata": metadata.to_dict(),
|
|
383
|
+
"resources": self._resources(),
|
|
377
384
|
}
|
|
378
385
|
|
|
379
386
|
def heartbeat(self) -> dict[str, Any]:
|
|
@@ -391,6 +398,7 @@ class ExecutionManager:
|
|
|
391
398
|
"activeExecutions": len(active),
|
|
392
399
|
"maxConcurrency": self.max_concurrent_executions,
|
|
393
400
|
"executions": active,
|
|
401
|
+
"resources": self._resources(),
|
|
394
402
|
}
|
|
395
403
|
|
|
396
404
|
def status(self, execution_id: str) -> dict[str, Any]:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ls-algorithm-plugin-sdk
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.7
|
|
4
4
|
Summary: Protocol-independent runtime SDK for dataset algorithms
|
|
5
5
|
Author: Ling Robotics
|
|
6
6
|
License: Proprietary
|
|
@@ -71,11 +71,19 @@ sudo /srv/camera-space-mano/.venv/bin/algorithm-plugin configure \
|
|
|
71
71
|
--api-key "$LDP_INTERNAL_API_KEY" \
|
|
72
72
|
--port 9030 \
|
|
73
73
|
--cluster production \
|
|
74
|
-
--gpu-ids 0,1
|
|
74
|
+
--gpu-ids 0,1 \
|
|
75
|
+
--log-retention-days 7 \
|
|
76
|
+
--output-retention-days 2 \
|
|
77
|
+
--scratch-retention-days 2
|
|
75
78
|
```
|
|
76
79
|
|
|
77
80
|
重复执行 configure 需要显式提供 `--force`。
|
|
78
81
|
|
|
82
|
+
配置部署的服务会启动独立清理进程,每小时扫描一次。仅删除创建超过日志保留期且
|
|
83
|
+
末行状态为 `succeeded` 的任务日志;`outputRoot` 和 `scratchRoot` 下超过对应保留期
|
|
84
|
+
未修改的数据也会被删除。清理记录按天写入 `<repository>/logs/cleanup-YYYYMMDD.log`,
|
|
85
|
+
包含清理时间、路径和字节数;清理记录自身不会被自动删除。
|
|
86
|
+
|
|
79
87
|
确认输出后执行:
|
|
80
88
|
|
|
81
89
|
```bash
|
|
@@ -2,6 +2,7 @@ README.md
|
|
|
2
2
|
pyproject.toml
|
|
3
3
|
src/algorithm_plugin_sdk/__init__.py
|
|
4
4
|
src/algorithm_plugin_sdk/algorithm.py
|
|
5
|
+
src/algorithm_plugin_sdk/cleanup.py
|
|
5
6
|
src/algorithm_plugin_sdk/cli.py
|
|
6
7
|
src/algorithm_plugin_sdk/context.py
|
|
7
8
|
src/algorithm_plugin_sdk/deployment.py
|
|
@@ -34,6 +35,7 @@ src/ls_algorithm_plugin_sdk.egg-info/dependency_links.txt
|
|
|
34
35
|
src/ls_algorithm_plugin_sdk.egg-info/entry_points.txt
|
|
35
36
|
src/ls_algorithm_plugin_sdk.egg-info/requires.txt
|
|
36
37
|
src/ls_algorithm_plugin_sdk.egg-info/top_level.txt
|
|
38
|
+
tests/test_cleanup.py
|
|
37
39
|
tests/test_cli.py
|
|
38
40
|
tests/test_configure.py
|
|
39
41
|
tests/test_cut.py
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import tempfile
|
|
4
|
+
import unittest
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from algorithm_plugin_sdk.cleanup import cleanup_once
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CleanupTests(unittest.TestCase):
|
|
12
|
+
def test_removes_only_expired_success_logs_and_stale_data(self) -> None:
|
|
13
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
14
|
+
root = Path(directory)
|
|
15
|
+
logs = root / "logs"
|
|
16
|
+
output = root / ".output"
|
|
17
|
+
scratch = root / ".scratch"
|
|
18
|
+
logs.mkdir()
|
|
19
|
+
output.mkdir()
|
|
20
|
+
scratch.mkdir()
|
|
21
|
+
now = datetime(2026, 9, 21, tzinfo=timezone.utc).timestamp()
|
|
22
|
+
|
|
23
|
+
success = logs / "20260901000000-success.log"
|
|
24
|
+
failed = logs / "20260901000000-failed.log"
|
|
25
|
+
recent = logs / "20260920000000-recent.log"
|
|
26
|
+
request = {
|
|
27
|
+
"event": "request",
|
|
28
|
+
"outputRoot": str(output),
|
|
29
|
+
"scratchRoot": str(scratch),
|
|
30
|
+
}
|
|
31
|
+
success.write_text(
|
|
32
|
+
json.dumps(request)
|
|
33
|
+
+ "\n"
|
|
34
|
+
+ json.dumps({**request, "outputRoot": "/"})
|
|
35
|
+
+ "\n"
|
|
36
|
+
+ json.dumps({"state": "succeeded"})
|
|
37
|
+
+ "\n",
|
|
38
|
+
encoding="utf-8",
|
|
39
|
+
)
|
|
40
|
+
failed.write_text(
|
|
41
|
+
json.dumps(request) + "\n" + json.dumps({"state": "failed"}) + "\n",
|
|
42
|
+
encoding="utf-8",
|
|
43
|
+
)
|
|
44
|
+
recent.write_text(
|
|
45
|
+
json.dumps(request) + "\n" + json.dumps({"state": "succeeded"}) + "\n",
|
|
46
|
+
encoding="utf-8",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
old_output = output / "old-job"
|
|
50
|
+
old_output.mkdir()
|
|
51
|
+
old_file = old_output / "result.bin"
|
|
52
|
+
old_file.write_bytes(b"result")
|
|
53
|
+
old_scratch = scratch / "old.tmp"
|
|
54
|
+
old_scratch.write_bytes(b"scratch")
|
|
55
|
+
recent_output = output / "recent-job"
|
|
56
|
+
recent_output.mkdir()
|
|
57
|
+
recent_file = recent_output / "result.bin"
|
|
58
|
+
recent_file.write_bytes(b"recent")
|
|
59
|
+
old_time = now - 3 * 86400
|
|
60
|
+
for path in (old_file, old_output, old_scratch):
|
|
61
|
+
os.utime(path, (old_time, old_time))
|
|
62
|
+
|
|
63
|
+
cleanup_once(logs, now=now)
|
|
64
|
+
|
|
65
|
+
self.assertFalse(success.exists())
|
|
66
|
+
self.assertTrue(failed.exists())
|
|
67
|
+
self.assertTrue(recent.exists())
|
|
68
|
+
self.assertFalse(old_output.exists())
|
|
69
|
+
self.assertFalse(old_scratch.exists())
|
|
70
|
+
self.assertTrue(recent_output.exists())
|
|
71
|
+
|
|
72
|
+
audit_logs = list(logs.glob("cleanup-*.log"))
|
|
73
|
+
self.assertEqual(len(audit_logs), 1)
|
|
74
|
+
entries = [
|
|
75
|
+
json.loads(line)
|
|
76
|
+
for line in audit_logs[0].read_text(encoding="utf-8").splitlines()
|
|
77
|
+
]
|
|
78
|
+
deleted = [entry for entry in entries if entry["event"] == "deleted"]
|
|
79
|
+
self.assertEqual(len(deleted), 3)
|
|
80
|
+
self.assertTrue(all("timestamp" in entry for entry in deleted))
|
|
81
|
+
self.assertTrue(all(entry["sizeBytes"] > 0 for entry in deleted))
|
|
82
|
+
self.assertEqual(entries[-1]["event"], "scan_completed")
|
|
83
|
+
self.assertIn("startedAt", entries[-1])
|
|
84
|
+
self.assertIn("finishedAt", entries[-1])
|
|
85
|
+
|
|
86
|
+
def test_never_deletes_cleanup_audit_logs(self) -> None:
|
|
87
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
88
|
+
logs = Path(directory)
|
|
89
|
+
audit = logs / "cleanup-20200101.log"
|
|
90
|
+
audit.write_text(json.dumps({"state": "succeeded"}) + "\n", encoding="utf-8")
|
|
91
|
+
|
|
92
|
+
cleanup_once(logs, log_retention_days=0)
|
|
93
|
+
|
|
94
|
+
self.assertTrue(audit.exists())
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
unittest.main()
|
|
@@ -142,6 +142,9 @@ class ConfigureTests(unittest.TestCase):
|
|
|
142
142
|
"--scratch-dir", "scratch",
|
|
143
143
|
"--local-scratch",
|
|
144
144
|
"--local-output",
|
|
145
|
+
"--log-retention-days", "8",
|
|
146
|
+
"--output-retention-days", "3",
|
|
147
|
+
"--scratch-retention-days", "4",
|
|
145
148
|
"--gpu-ids", "2,0",
|
|
146
149
|
"--service-token", "secret",
|
|
147
150
|
"--default-parameters", '{"batchSize": 8, "mode": "fast"}',
|
|
@@ -159,6 +162,9 @@ class ConfigureTests(unittest.TestCase):
|
|
|
159
162
|
self.assertEqual(args.scratch_dir, "scratch")
|
|
160
163
|
self.assertTrue(args.local_scratch)
|
|
161
164
|
self.assertTrue(args.local_output)
|
|
165
|
+
self.assertEqual(args.log_retention_days, 8)
|
|
166
|
+
self.assertEqual(args.output_retention_days, 3)
|
|
167
|
+
self.assertEqual(args.scratch_retention_days, 4)
|
|
162
168
|
self.assertEqual(args.gpu_ids, [2, 0])
|
|
163
169
|
self.assertEqual(args.service_token, "secret")
|
|
164
170
|
self.assertEqual(
|
|
@@ -220,6 +226,9 @@ class ConfigureTests(unittest.TestCase):
|
|
|
220
226
|
gpu_ids=[3, 1],
|
|
221
227
|
environment={"DEMO_MODE": "fast"},
|
|
222
228
|
default_parameters={},
|
|
229
|
+
log_retention_days=7,
|
|
230
|
+
output_retention_days=2,
|
|
231
|
+
scratch_retention_days=2,
|
|
223
232
|
)
|
|
224
233
|
config.save.assert_called_once_with()
|
|
225
234
|
|
|
@@ -87,6 +87,9 @@ class DeploymentConfigTests(unittest.TestCase):
|
|
|
87
87
|
self.assertEqual(loaded.payload["service"]["gpuIds"], [3, 1])
|
|
88
88
|
self.assertTrue(loaded.payload["service"]["localScratch"])
|
|
89
89
|
self.assertTrue(loaded.payload["service"]["localOutput"])
|
|
90
|
+
self.assertEqual(loaded.payload["service"]["logRetentionDays"], 7)
|
|
91
|
+
self.assertEqual(loaded.payload["service"]["outputRetentionDays"], 2)
|
|
92
|
+
self.assertEqual(loaded.payload["service"]["scratchRetentionDays"], 2)
|
|
90
93
|
manifest_path = root / "release-manifest.json"
|
|
91
94
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
92
95
|
self.assertEqual(
|
|
@@ -142,6 +145,25 @@ class DeploymentConfigTests(unittest.TestCase):
|
|
|
142
145
|
)
|
|
143
146
|
self.assertEqual(registration["namespace"], "ignored")
|
|
144
147
|
|
|
148
|
+
def test_old_service_config_receives_retention_defaults(self) -> None:
|
|
149
|
+
config = DeploymentConfig(
|
|
150
|
+
Path("/tmp/algorithm-plugin.json"),
|
|
151
|
+
{
|
|
152
|
+
"service": {
|
|
153
|
+
"port": 9000,
|
|
154
|
+
"maxConcurrency": 1,
|
|
155
|
+
"gpuIds": [],
|
|
156
|
+
"defaultParameters": {},
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
config._validate_service()
|
|
162
|
+
|
|
163
|
+
self.assertEqual(config.payload["service"]["logRetentionDays"], 7)
|
|
164
|
+
self.assertEqual(config.payload["service"]["outputRetentionDays"], 2)
|
|
165
|
+
self.assertEqual(config.payload["service"]["scratchRetentionDays"], 2)
|
|
166
|
+
|
|
145
167
|
def test_repository_is_importable_during_generate_and_load(self) -> None:
|
|
146
168
|
with tempfile.TemporaryDirectory() as directory:
|
|
147
169
|
root = Path(directory)
|
|
@@ -43,7 +43,7 @@ class PluginRegistrationAgentTests(unittest.TestCase):
|
|
|
43
43
|
"status": "ready",
|
|
44
44
|
"activeExecutions": 0,
|
|
45
45
|
"maxConcurrency": 1,
|
|
46
|
-
"resources": {},
|
|
46
|
+
"resources": {"gpuIds": [0, 1]},
|
|
47
47
|
}
|
|
48
48
|
agent = PluginRegistrationAgent(
|
|
49
49
|
settings,
|
|
@@ -64,6 +64,10 @@ class PluginRegistrationAgentTests(unittest.TestCase):
|
|
|
64
64
|
self.assertNotIn("implementationDigest", registration["manifest"])
|
|
65
65
|
self.assertNotIn("metadata", registration["manifest"])
|
|
66
66
|
self.assertTrue(client.calls[1][0].endswith("/instance-1/heartbeat"))
|
|
67
|
+
self.assertEqual(
|
|
68
|
+
client.calls[1][1]["resources"],
|
|
69
|
+
{"gpuIds": [0, 1]},
|
|
70
|
+
)
|
|
67
71
|
|
|
68
72
|
def test_default_instance_key_is_stable_and_not_release_specific(self) -> None:
|
|
69
73
|
first = stable_instance_key("camera_space_mano")
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/tests/test_sdk_automation.py
RENAMED
|
@@ -24,12 +24,16 @@ class SdkAutomationTests(unittest.TestCase):
|
|
|
24
24
|
manager = ExecutionManager(
|
|
25
25
|
AlgorithmRunner(RecordingAlgorithm()),
|
|
26
26
|
release_manifest=release,
|
|
27
|
+
gpu_ids=[0, 1, 2, 3, 4, 5, 6],
|
|
27
28
|
)
|
|
28
29
|
manifest = manager.manifest()
|
|
29
30
|
self.assertEqual(manifest["implementationDigest"], f"sha256:{'2' * 64}")
|
|
30
31
|
self.assertTrue(manifest["capabilities"]["cli"])
|
|
31
32
|
self.assertTrue(manifest["capabilities"]["webui"])
|
|
32
33
|
self.assertTrue(manifest["capabilities"]["httpService"])
|
|
34
|
+
resources = {"gpuIds": [0, 1, 2, 3, 4, 5, 6]}
|
|
35
|
+
self.assertEqual(manifest["resources"], resources)
|
|
36
|
+
self.assertEqual(manager.heartbeat()["resources"], resources)
|
|
33
37
|
manager.close()
|
|
34
38
|
|
|
35
39
|
def test_registration_is_inactive_when_no_deployment_is_configured(self) -> None:
|
|
@@ -28,6 +28,9 @@ class ServeTests(unittest.TestCase):
|
|
|
28
28
|
"gpuIds": [0],
|
|
29
29
|
"webui": False,
|
|
30
30
|
"defaultParameters": {},
|
|
31
|
+
"logRetentionDays": 7,
|
|
32
|
+
"outputRetentionDays": 2,
|
|
33
|
+
"scratchRetentionDays": 2,
|
|
31
34
|
}
|
|
32
35
|
}
|
|
33
36
|
config.process_environment.return_value = dict(os.environ)
|
|
@@ -59,6 +62,7 @@ class ServeTests(unittest.TestCase):
|
|
|
59
62
|
return_value=None,
|
|
60
63
|
),
|
|
61
64
|
patch("algorithm_plugin_sdk.cli_impl.serve.run_server"),
|
|
65
|
+
patch("algorithm_plugin_sdk.cli_impl.serve.CleanupProcess") as cleanup_class,
|
|
62
66
|
patch("algorithm_plugin_sdk.cli_impl.serve.os.chdir"),
|
|
63
67
|
):
|
|
64
68
|
self.assertEqual(execute(args), 0)
|
|
@@ -67,6 +71,8 @@ class ServeTests(unittest.TestCase):
|
|
|
67
71
|
self.assertTrue(manager_kwargs["local_scratch"])
|
|
68
72
|
self.assertTrue(manager_kwargs["local_output"])
|
|
69
73
|
self.assertEqual(manager_kwargs["repository"], config.repository)
|
|
74
|
+
cleanup_class.return_value.start.assert_called_once_with()
|
|
75
|
+
cleanup_class.return_value.stop.assert_called_once_with()
|
|
70
76
|
|
|
71
77
|
|
|
72
78
|
if __name__ == "__main__":
|
|
@@ -40,6 +40,16 @@ class ExecutionManagerTests(unittest.TestCase):
|
|
|
40
40
|
manager.close()
|
|
41
41
|
self.assertEqual(algorithm.calls[0].gpu_ids, [3, 1])
|
|
42
42
|
|
|
43
|
+
def test_reports_configured_gpu_resources(self) -> None:
|
|
44
|
+
manager = ExecutionManager(
|
|
45
|
+
AlgorithmRunner(RecordingAlgorithm()),
|
|
46
|
+
gpu_ids=[0, 1, 2, 3, 4, 5, 6],
|
|
47
|
+
)
|
|
48
|
+
resources = {"gpuIds": [0, 1, 2, 3, 4, 5, 6]}
|
|
49
|
+
self.assertEqual(manager.manifest()["resources"], resources)
|
|
50
|
+
self.assertEqual(manager.heartbeat()["resources"], resources)
|
|
51
|
+
manager.close()
|
|
52
|
+
|
|
43
53
|
def test_fills_missing_parameters_from_configured_defaults(self) -> None:
|
|
44
54
|
algorithm = RecordingAlgorithm()
|
|
45
55
|
manager = ExecutionManager(
|
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/cli.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/context.py
RENAMED
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/errors.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/loader.py
RENAMED
|
File without changes
|
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/models.py
RENAMED
|
File without changes
|
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/release.py
RENAMED
|
File without changes
|
{ls_algorithm_plugin_sdk-0.3.6 → ls_algorithm_plugin_sdk-0.3.7}/src/algorithm_plugin_sdk/runner.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|