sediment-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sediment_api/__init__.py +6 -0
- sediment_api/config.py +230 -0
- sediment_api/database.py +43 -0
- sediment_api/deps.py +191 -0
- sediment_api/main.py +212 -0
- sediment_api/mirror_gc.py +151 -0
- sediment_api/reports/__init__.py +9 -0
- sediment_api/reports/abandonment_report.py +123 -0
- sediment_api/reports/attribution_share_report.py +298 -0
- sediment_api/reports/dataset_diagnostics.py +311 -0
- sediment_api/reports/label_confidence_inspection.py +438 -0
- sediment_api/reports/lifecycle_report.py +73 -0
- sediment_api/reports/merge_retention_report.py +165 -0
- sediment_api/reports/model_report.py +1534 -0
- sediment_api/reports/precision_report.py +182 -0
- sediment_api/reports/recovery_yield_report.py +132 -0
- sediment_api/routers/__init__.py +3 -0
- sediment_api/routers/ci_vendor.py +149 -0
- sediment_api/routers/forge.py +452 -0
- sediment_api/routers/gateway.py +137 -0
- sediment_api/routers/otlp.py +120 -0
- sediment_api/routers/query.py +1010 -0
- sediment_api/routers/reports.py +116 -0
- sediment_api/routers/v1.py +170 -0
- sediment_api/services/__init__.py +2 -0
- sediment_api/services/operational_reports.py +391 -0
- sediment_api/worker.py +242 -0
- sediment_api/workers.py +382 -0
- sediment_api-0.1.0.dist-info/METADATA +20 -0
- sediment_api-0.1.0.dist-info/RECORD +32 -0
- sediment_api-0.1.0.dist-info/WHEEL +4 -0
- sediment_api-0.1.0.dist-info/licenses/LICENSE +661 -0
sediment_api/worker.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""One fixed API operation in a disposable process; stdout is bounded IPC."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from dataclasses import asdict, dataclass
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import sys
|
|
10
|
+
from typing import Any, Literal
|
|
11
|
+
|
|
12
|
+
from fastapi import HTTPException
|
|
13
|
+
from fastapi.responses import JSONResponse, Response
|
|
14
|
+
from pydantic import TypeAdapter
|
|
15
|
+
from sediment_core import FactStore, NonEmptyId, OperationalReportLimitExceeded, Push
|
|
16
|
+
from sediment_core.models import AwareDatetime, CommitSha, RepoSlug, RequiredRepoSlug
|
|
17
|
+
from sediment_core.postgres_engine import create_postgres_engine
|
|
18
|
+
from sediment_derive import MirrorManager
|
|
19
|
+
from sediment_derive.repository_identity import (
|
|
20
|
+
REPOSITORY_IDENTITY_SKIP_REASONS,
|
|
21
|
+
RepositoryIdentity,
|
|
22
|
+
)
|
|
23
|
+
from sqlalchemy.exc import (
|
|
24
|
+
DisconnectionError,
|
|
25
|
+
SQLAlchemyError,
|
|
26
|
+
OperationalError,
|
|
27
|
+
InterfaceError,
|
|
28
|
+
TimeoutError as PoolTimeout,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
from .config import settings
|
|
32
|
+
from .workers import MAX_REQUEST_BYTES
|
|
33
|
+
from .routers import forge, query, reports
|
|
34
|
+
from .services.operational_reports import (
|
|
35
|
+
LifecycleReportRequest,
|
|
36
|
+
generate_lifecycle_report,
|
|
37
|
+
model_report_payload,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class _Diagnostics(logging.Handler):
|
|
42
|
+
"""Send closed counters, never raw errors or exception tracebacks, to stderr."""
|
|
43
|
+
|
|
44
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
45
|
+
if (
|
|
46
|
+
record.msg
|
|
47
|
+
== "attributions_derived repo=%s attributed=%d notes=%d jaccard=%d"
|
|
48
|
+
):
|
|
49
|
+
repo, attributed, notes, jaccard = record.args
|
|
50
|
+
value = {
|
|
51
|
+
"event": "attributions_derived",
|
|
52
|
+
"repo": repo,
|
|
53
|
+
"attributed": attributed,
|
|
54
|
+
"notes": notes,
|
|
55
|
+
"jaccard": jaccard,
|
|
56
|
+
}
|
|
57
|
+
elif (
|
|
58
|
+
record.msg
|
|
59
|
+
== "session_commit_observations_captured repo=%s stored=%d duplicates=%d"
|
|
60
|
+
):
|
|
61
|
+
repo, stored, duplicates = record.args
|
|
62
|
+
value = {
|
|
63
|
+
"event": "session_commit_observations_captured",
|
|
64
|
+
"repo": repo,
|
|
65
|
+
"stored": stored,
|
|
66
|
+
"duplicates": duplicates,
|
|
67
|
+
}
|
|
68
|
+
elif (
|
|
69
|
+
record.msg == "mirror_refresh_skipped repo=%s reason=%s count=1"
|
|
70
|
+
and str(record.args[1]) in REPOSITORY_IDENTITY_SKIP_REASONS
|
|
71
|
+
):
|
|
72
|
+
value = {
|
|
73
|
+
"event": "repository_identity_declined",
|
|
74
|
+
"repo": record.args[0],
|
|
75
|
+
"reason": str(record.args[1]),
|
|
76
|
+
"count": 1,
|
|
77
|
+
}
|
|
78
|
+
elif record.levelno >= logging.WARNING:
|
|
79
|
+
value = {"event": "worker_operation_warning"}
|
|
80
|
+
else:
|
|
81
|
+
return
|
|
82
|
+
sys.stderr.write(json.dumps(value, ensure_ascii=True) + "\n")
|
|
83
|
+
sys.stderr.flush()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class WorkerRequest:
|
|
88
|
+
"""Private subprocess envelope; captured Facts retain their canonical models."""
|
|
89
|
+
|
|
90
|
+
kind: Literal[
|
|
91
|
+
"commit", "session", "model-report", "lifecycle-report", "push", "rename"
|
|
92
|
+
]
|
|
93
|
+
payload: dict[str, Any]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _dispatch(request: WorkerRequest, store: FactStore) -> Response:
|
|
97
|
+
payload = request.payload
|
|
98
|
+
match request.kind:
|
|
99
|
+
case "commit":
|
|
100
|
+
sha = TypeAdapter(CommitSha).validate_python(payload["sha"])
|
|
101
|
+
options = {}
|
|
102
|
+
for name, shape in (
|
|
103
|
+
("repo", RequiredRepoSlug),
|
|
104
|
+
("repository_identity", RepositoryIdentity),
|
|
105
|
+
("as_of", AwareDatetime),
|
|
106
|
+
):
|
|
107
|
+
if payload.get(name) is not None:
|
|
108
|
+
options[name] = TypeAdapter(shape).validate_python(payload[name])
|
|
109
|
+
try:
|
|
110
|
+
value = query._run_query(sha, store, **options)
|
|
111
|
+
except OperationalReportLimitExceeded:
|
|
112
|
+
raise HTTPException(
|
|
113
|
+
status_code=409, detail={"reason": "repository_evidence_limit"}
|
|
114
|
+
) from None
|
|
115
|
+
return query._query_response(value, dict[str, Any])
|
|
116
|
+
case "session":
|
|
117
|
+
session_id = TypeAdapter(NonEmptyId).validate_python(payload["session_id"])
|
|
118
|
+
try:
|
|
119
|
+
value = query._run_session_query(session_id, store)
|
|
120
|
+
except OperationalReportLimitExceeded:
|
|
121
|
+
raise HTTPException(
|
|
122
|
+
status_code=409, detail={"reason": "repository_evidence_limit"}
|
|
123
|
+
) from None
|
|
124
|
+
except ValueError as exc:
|
|
125
|
+
raise HTTPException(status_code=409, detail=str(exc)) from None
|
|
126
|
+
return query._query_response(
|
|
127
|
+
value,
|
|
128
|
+
query.SessionDossierFound | query.SessionDossierNotFound,
|
|
129
|
+
exclude_none=True,
|
|
130
|
+
)
|
|
131
|
+
case "model-report" | "lifecycle-report":
|
|
132
|
+
bounds = reports.ReportScopeResponse.model_validate(payload)
|
|
133
|
+
scope = reports._build_scope(
|
|
134
|
+
bounds.cohort_start, bounds.cohort_end, bounds.as_of
|
|
135
|
+
)
|
|
136
|
+
mirrors = MirrorManager(settings.mirror_path or "./mirrors")
|
|
137
|
+
try:
|
|
138
|
+
if request.kind == "model-report":
|
|
139
|
+
result = model_report_payload(
|
|
140
|
+
store, mirrors, settings.org_id, scope
|
|
141
|
+
)
|
|
142
|
+
else:
|
|
143
|
+
result = asdict(
|
|
144
|
+
generate_lifecycle_report(
|
|
145
|
+
store,
|
|
146
|
+
mirrors,
|
|
147
|
+
LifecycleReportRequest(settings.org_id, scope),
|
|
148
|
+
).report
|
|
149
|
+
)
|
|
150
|
+
except OperationalReportLimitExceeded:
|
|
151
|
+
raise HTTPException(
|
|
152
|
+
status_code=409, detail="report evidence exceeds the fixed limit"
|
|
153
|
+
) from None
|
|
154
|
+
return reports._report_response(
|
|
155
|
+
reports.ReportEnvelope(scope=bounds, report=result)
|
|
156
|
+
)
|
|
157
|
+
case "push":
|
|
158
|
+
push = Push.model_validate(payload["push"])
|
|
159
|
+
if push.org_id != settings.org_id:
|
|
160
|
+
raise ValueError("worker organization mismatch")
|
|
161
|
+
forge._refresh_mirror(
|
|
162
|
+
push,
|
|
163
|
+
store,
|
|
164
|
+
fetch_repo=TypeAdapter(RepoSlug | None).validate_python(
|
|
165
|
+
payload.get("fetch_repo")
|
|
166
|
+
),
|
|
167
|
+
fetch_clone_url=TypeAdapter(str | None).validate_python(
|
|
168
|
+
payload.get("fetch_clone_url")
|
|
169
|
+
),
|
|
170
|
+
)
|
|
171
|
+
return JSONResponse({"completed": True})
|
|
172
|
+
case "rename":
|
|
173
|
+
old_repo = TypeAdapter(RepoSlug).validate_python(payload["old_repo"])
|
|
174
|
+
new_repo = TypeAdapter(RepoSlug).validate_python(payload["new_repo"])
|
|
175
|
+
if not old_repo or not new_repo or not settings.mirror_path:
|
|
176
|
+
raise ValueError("mirror rename requires configured repository paths")
|
|
177
|
+
moved = MirrorManager(settings.mirror_path).rename(
|
|
178
|
+
settings.org_id, old_repo, new_repo
|
|
179
|
+
)
|
|
180
|
+
return JSONResponse({"renamed": moved})
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def execute(request: WorkerRequest) -> Response:
|
|
184
|
+
engine = None
|
|
185
|
+
try:
|
|
186
|
+
engine = create_postgres_engine(
|
|
187
|
+
settings.database_url.get_secret_value(),
|
|
188
|
+
api_work=True,
|
|
189
|
+
single_connection=True,
|
|
190
|
+
)
|
|
191
|
+
return _dispatch(request, FactStore(engine))
|
|
192
|
+
except HTTPException as exc:
|
|
193
|
+
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
|
|
194
|
+
except (DisconnectionError, OperationalError, InterfaceError, PoolTimeout):
|
|
195
|
+
return JSONResponse(
|
|
196
|
+
status_code=503,
|
|
197
|
+
content={
|
|
198
|
+
"detail": {
|
|
199
|
+
"code": "database_unavailable",
|
|
200
|
+
"message": "PostgreSQL fact store unavailable",
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
)
|
|
204
|
+
except SQLAlchemyError:
|
|
205
|
+
return JSONResponse(
|
|
206
|
+
status_code=500,
|
|
207
|
+
content={
|
|
208
|
+
"detail": {
|
|
209
|
+
"code": "database_operation_failed",
|
|
210
|
+
"message": "PostgreSQL fact store operation failed",
|
|
211
|
+
}
|
|
212
|
+
},
|
|
213
|
+
)
|
|
214
|
+
except Exception:
|
|
215
|
+
return JSONResponse(status_code=500, content={"detail": "work failed"})
|
|
216
|
+
finally:
|
|
217
|
+
if engine is not None:
|
|
218
|
+
engine.dispose()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def main() -> int:
|
|
222
|
+
logging.basicConfig(level=logging.INFO, handlers=[_Diagnostics()], force=True)
|
|
223
|
+
try:
|
|
224
|
+
data = sys.stdin.buffer.read(MAX_REQUEST_BYTES + 1)
|
|
225
|
+
if len(data) > MAX_REQUEST_BYTES:
|
|
226
|
+
return 1
|
|
227
|
+
envelope = json.loads(data)
|
|
228
|
+
if not isinstance(envelope, dict) or set(envelope) != {"kind", "payload"}:
|
|
229
|
+
return 1
|
|
230
|
+
request = TypeAdapter(WorkerRequest).validate_python(envelope)
|
|
231
|
+
response = execute(request)
|
|
232
|
+
sys.stdout.buffer.write(str(response.status_code).encode() + b"\n")
|
|
233
|
+
sys.stdout.buffer.write(response.body)
|
|
234
|
+
sys.stdout.buffer.flush()
|
|
235
|
+
return 0
|
|
236
|
+
except Exception:
|
|
237
|
+
# No traceback or request/driver diagnostics cross the process boundary.
|
|
238
|
+
return 1
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
if __name__ == "__main__":
|
|
242
|
+
raise SystemExit(main())
|
sediment_api/workers.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Bounded, disposable processes for the API's six expensive operations."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import ctypes
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import shutil
|
|
13
|
+
import signal
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
from fastapi import HTTPException
|
|
17
|
+
from fastapi.responses import JSONResponse, Response
|
|
18
|
+
from sediment_core import Push, normalize_repo_slug
|
|
19
|
+
from sediment_derive.repository_identity import REPOSITORY_IDENTITY_SKIP_REASONS
|
|
20
|
+
|
|
21
|
+
from .config import settings
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger("sediment.api.workers")
|
|
24
|
+
|
|
25
|
+
QUERY_SLOTS = 2
|
|
26
|
+
MIRROR_SLOTS = 2
|
|
27
|
+
MIRROR_QUEUE_SIZE = 16
|
|
28
|
+
QUERY_BUDGET_SECONDS = 30.0
|
|
29
|
+
MIRROR_BUDGET_SECONDS = 120.0
|
|
30
|
+
TERMINATION_GRACE_SECONDS = 1.0
|
|
31
|
+
MAX_RESULT_BYTES = 64 * 1024 * 1024
|
|
32
|
+
MAX_DIAGNOSTIC_BYTES = 64 * 1024
|
|
33
|
+
MAX_REQUEST_BYTES = 1024 * 1024
|
|
34
|
+
MIRROR_FREE_BYTES = 1024 * 1024 * 1024
|
|
35
|
+
_WORKER_COMMAND = (sys.executable, "-m", "sediment_api.worker")
|
|
36
|
+
_QUERY_KINDS = frozenset({"commit", "session", "model-report", "lifecycle-report"})
|
|
37
|
+
_MIRROR_KINDS = frozenset({"push", "rename"})
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _unavailable(reason: str) -> Response:
|
|
41
|
+
return JSONResponse(status_code=503, content={"detail": reason})
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _group_exists(pid: int) -> bool:
|
|
45
|
+
try:
|
|
46
|
+
os.killpg(pid, 0)
|
|
47
|
+
return True
|
|
48
|
+
except ProcessLookupError:
|
|
49
|
+
return False
|
|
50
|
+
except PermissionError:
|
|
51
|
+
# Darwin can report EPERM while the last member exits. This still
|
|
52
|
+
# means "possibly present": keep checking instead of releasing a slot.
|
|
53
|
+
return True
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _signal_group(pid: int, sig: signal.Signals) -> None:
|
|
57
|
+
try:
|
|
58
|
+
os.killpg(pid, sig)
|
|
59
|
+
except (ProcessLookupError, PermissionError):
|
|
60
|
+
# Cleanup keeps checking/reaping the group, including this exit race.
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _reap_descendants(pid: int) -> None:
|
|
65
|
+
# The direct child belongs to asyncio's watcher; call this only after wait().
|
|
66
|
+
while True:
|
|
67
|
+
try:
|
|
68
|
+
child, _ = os.waitpid(-pid, os.WNOHANG)
|
|
69
|
+
except ChildProcessError:
|
|
70
|
+
return
|
|
71
|
+
if child == 0:
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
async def _terminate(process: asyncio.subprocess.Process) -> None:
|
|
76
|
+
"""Stop the complete process group, including descendants left on success."""
|
|
77
|
+
_signal_group(process.pid, signal.SIGTERM)
|
|
78
|
+
deadline = asyncio.get_running_loop().time() + TERMINATION_GRACE_SECONDS
|
|
79
|
+
while _group_exists(process.pid):
|
|
80
|
+
if process.returncode is not None:
|
|
81
|
+
await process.wait()
|
|
82
|
+
_reap_descendants(process.pid)
|
|
83
|
+
if asyncio.get_running_loop().time() >= deadline:
|
|
84
|
+
_signal_group(process.pid, signal.SIGKILL)
|
|
85
|
+
break
|
|
86
|
+
await asyncio.sleep(0.01)
|
|
87
|
+
await process.wait()
|
|
88
|
+
# Linux reparents owned grandchildren here because the supervisor is a
|
|
89
|
+
# subreaper. On macOS the system reaper owns them. Never reap another group.
|
|
90
|
+
while _group_exists(process.pid):
|
|
91
|
+
_reap_descendants(process.pid)
|
|
92
|
+
await asyncio.sleep(0.01)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
async def _finish_cleanup(task: asyncio.Task) -> None:
|
|
96
|
+
"""Repeated caller cancellation must not abandon process cleanup."""
|
|
97
|
+
while not task.done():
|
|
98
|
+
try:
|
|
99
|
+
await asyncio.shield(task)
|
|
100
|
+
except asyncio.CancelledError:
|
|
101
|
+
continue
|
|
102
|
+
task.result()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class _OutputLimit(Exception):
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class _WorkerFailure(Exception):
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
async def _read_result(stream: asyncio.StreamReader) -> Response:
|
|
114
|
+
# Status line precedes the raw response body, avoiding a base64 copy of a
|
|
115
|
+
# 64 MiB report. No worker-controlled response headers cross this boundary.
|
|
116
|
+
status = await stream.readline()
|
|
117
|
+
if status not in {b"200\n", b"409\n", b"422\n", b"500\n", b"503\n"}:
|
|
118
|
+
raise _WorkerFailure
|
|
119
|
+
body = bytearray()
|
|
120
|
+
while chunk := await stream.read(64 * 1024):
|
|
121
|
+
if len(body) + len(chunk) > MAX_RESULT_BYTES:
|
|
122
|
+
raise _OutputLimit
|
|
123
|
+
body.extend(chunk)
|
|
124
|
+
try:
|
|
125
|
+
# Decline malformed IPC rather than forwarding arbitrary worker stdout.
|
|
126
|
+
json.loads(body)
|
|
127
|
+
except (ValueError, RecursionError):
|
|
128
|
+
raise _WorkerFailure from None
|
|
129
|
+
return Response(bytes(body), status_code=int(status), media_type="application/json")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def _read_diagnostics(stream: asyncio.StreamReader) -> None:
|
|
133
|
+
received = 0
|
|
134
|
+
while chunk := await stream.readline():
|
|
135
|
+
received += len(chunk)
|
|
136
|
+
if received > MAX_DIAGNOSTIC_BYTES:
|
|
137
|
+
raise _WorkerFailure
|
|
138
|
+
# Only the two closed count records cross into parent logs. Raw stderr,
|
|
139
|
+
# exception text, and driver/Git diagnostics stay outside the API log.
|
|
140
|
+
try:
|
|
141
|
+
value = json.loads(chunk)
|
|
142
|
+
event = value.get("event")
|
|
143
|
+
repo = normalize_repo_slug(value.get("repo", ""))
|
|
144
|
+
if event == "attributions_derived" and all(
|
|
145
|
+
type(value.get(key)) is int and value[key] >= 0
|
|
146
|
+
for key in ("attributed", "notes", "jaccard")
|
|
147
|
+
):
|
|
148
|
+
logging.getLogger("sediment.api.forge").info(
|
|
149
|
+
"attributions_derived repo=%s attributed=%d notes=%d jaccard=%d",
|
|
150
|
+
repo,
|
|
151
|
+
value["attributed"],
|
|
152
|
+
value["notes"],
|
|
153
|
+
value["jaccard"],
|
|
154
|
+
)
|
|
155
|
+
elif event == "session_commit_observations_captured" and all(
|
|
156
|
+
type(value.get(key)) is int and value[key] >= 0
|
|
157
|
+
for key in ("stored", "duplicates")
|
|
158
|
+
):
|
|
159
|
+
logging.getLogger("sediment.api.forge").info(
|
|
160
|
+
"session_commit_observations_captured repo=%s stored=%d duplicates=%d",
|
|
161
|
+
repo,
|
|
162
|
+
value["stored"],
|
|
163
|
+
value["duplicates"],
|
|
164
|
+
)
|
|
165
|
+
elif (
|
|
166
|
+
event == "repository_identity_declined"
|
|
167
|
+
and value.get("reason") in REPOSITORY_IDENTITY_SKIP_REASONS
|
|
168
|
+
and type(value.get("count")) is int
|
|
169
|
+
and value["count"] == 1
|
|
170
|
+
):
|
|
171
|
+
logging.getLogger("sediment.api.forge").warning(
|
|
172
|
+
"repository_identity_declined repo=%s reason=%s count=1",
|
|
173
|
+
repo,
|
|
174
|
+
value["reason"],
|
|
175
|
+
)
|
|
176
|
+
elif event == "worker_operation_warning":
|
|
177
|
+
logger.warning("worker_operation_warning")
|
|
178
|
+
except (AttributeError, TypeError, ValueError):
|
|
179
|
+
continue
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
async def _discard(stream: asyncio.StreamReader) -> None:
|
|
183
|
+
while await stream.read(64 * 1024):
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _child_environment() -> dict[str, str]:
|
|
188
|
+
environment = dict(os.environ)
|
|
189
|
+
# Capture validated parent settings for each job. Database credentials never
|
|
190
|
+
# enter command arguments, the pipe envelope, or diagnostic messages.
|
|
191
|
+
environment.update(
|
|
192
|
+
SEDIMENT_DATABASE_URL=settings.database_url.get_secret_value(),
|
|
193
|
+
SEDIMENT_ORG_ID=settings.org_id,
|
|
194
|
+
SEDIMENT_MIRROR_PATH=settings.mirror_path or "",
|
|
195
|
+
SEDIMENT_DEV_MODE=str(settings.dev_mode).lower(),
|
|
196
|
+
SEDIMENT_ALLOWED_CLONE_HOSTS=json.dumps(settings.allowed_clone_hosts),
|
|
197
|
+
)
|
|
198
|
+
return environment
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _request_data(kind: str, payload: dict) -> bytes:
|
|
202
|
+
request = json.dumps({"kind": kind, "payload": payload}).encode()
|
|
203
|
+
if len(request) > MAX_REQUEST_BYTES:
|
|
204
|
+
raise HTTPException(
|
|
205
|
+
status_code=503, detail="work request exceeds the fixed limit"
|
|
206
|
+
)
|
|
207
|
+
return request
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
async def _execute(request: bytes) -> Response:
|
|
211
|
+
process = None
|
|
212
|
+
readers: list[asyncio.Task] = []
|
|
213
|
+
started = asyncio.create_task(
|
|
214
|
+
asyncio.create_subprocess_exec(
|
|
215
|
+
*_WORKER_COMMAND,
|
|
216
|
+
stdin=asyncio.subprocess.PIPE,
|
|
217
|
+
stdout=asyncio.subprocess.PIPE,
|
|
218
|
+
stderr=asyncio.subprocess.PIPE,
|
|
219
|
+
start_new_session=True,
|
|
220
|
+
limit=64 * 1024,
|
|
221
|
+
env=_child_environment(),
|
|
222
|
+
)
|
|
223
|
+
)
|
|
224
|
+
try:
|
|
225
|
+
# Cancellation during spawn still owns the process that spawn creates.
|
|
226
|
+
process = await asyncio.shield(started)
|
|
227
|
+
readers = [
|
|
228
|
+
asyncio.create_task(_read_result(process.stdout)),
|
|
229
|
+
asyncio.create_task(_read_diagnostics(process.stderr)),
|
|
230
|
+
]
|
|
231
|
+
process.stdin.write(request)
|
|
232
|
+
await process.stdin.drain()
|
|
233
|
+
process.stdin.close()
|
|
234
|
+
result, _ = await asyncio.gather(*readers)
|
|
235
|
+
if await process.wait() != 0:
|
|
236
|
+
raise _WorkerFailure
|
|
237
|
+
return result
|
|
238
|
+
finally:
|
|
239
|
+
|
|
240
|
+
async def cleanup() -> None:
|
|
241
|
+
nonlocal process
|
|
242
|
+
if process is None:
|
|
243
|
+
# Shielded spawn may finish after its caller was cancelled.
|
|
244
|
+
process = await started
|
|
245
|
+
for reader in readers:
|
|
246
|
+
reader.cancel()
|
|
247
|
+
await asyncio.gather(*readers, return_exceptions=True)
|
|
248
|
+
# Drain pipes during termination even after a cap/failure stopped
|
|
249
|
+
# their readers. A paused full pipe otherwise prevents EOF and can
|
|
250
|
+
# keep asyncio's subprocess transport alive after wait().
|
|
251
|
+
process.stdin.close()
|
|
252
|
+
await asyncio.gather(
|
|
253
|
+
_terminate(process),
|
|
254
|
+
_discard(process.stdout),
|
|
255
|
+
_discard(process.stderr),
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
await _finish_cleanup(asyncio.create_task(cleanup()))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
class WorkerSupervisor:
|
|
262
|
+
"""Own transient admission and child lifetimes for one API process."""
|
|
263
|
+
|
|
264
|
+
def __init__(self) -> None:
|
|
265
|
+
if sys.platform == "linux":
|
|
266
|
+
# PR_SET_CHILD_SUBREAPER: adopt orphaned Git descendants so they
|
|
267
|
+
# cannot accumulate under a container PID 1 that does not reap them.
|
|
268
|
+
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
|
269
|
+
raise RuntimeError("cannot initialize worker process cleanup")
|
|
270
|
+
self._query_tasks: set[asyncio.Task] = set()
|
|
271
|
+
self._mirror_tasks: set[asyncio.Task] = set()
|
|
272
|
+
self._push_tasks: dict[tuple[str, str], asyncio.Task] = {}
|
|
273
|
+
self._mirror_slots = asyncio.Semaphore(MIRROR_SLOTS)
|
|
274
|
+
self._closed = False
|
|
275
|
+
|
|
276
|
+
async def _run(self, kind: str, request: bytes, *, mirror: bool) -> Response:
|
|
277
|
+
budget = MIRROR_BUDGET_SECONDS if mirror else QUERY_BUDGET_SECONDS
|
|
278
|
+
try:
|
|
279
|
+
async with asyncio.timeout(budget):
|
|
280
|
+
if mirror:
|
|
281
|
+
async with self._mirror_slots:
|
|
282
|
+
return await _execute(request)
|
|
283
|
+
return await _execute(request)
|
|
284
|
+
except TimeoutError:
|
|
285
|
+
logger.warning("worker_budget_exceeded kind=%s", kind)
|
|
286
|
+
return _unavailable("work budget exceeded")
|
|
287
|
+
except _OutputLimit:
|
|
288
|
+
logger.warning("worker_output_limit_exceeded kind=%s", kind)
|
|
289
|
+
return JSONResponse(
|
|
290
|
+
status_code=409, content={"detail": "response exceeds the fixed limit"}
|
|
291
|
+
)
|
|
292
|
+
except (OSError, ValueError, _WorkerFailure) as error:
|
|
293
|
+
logger.error("worker_failed kind=%s reason=%s", kind, type(error).__name__)
|
|
294
|
+
return _unavailable("work unavailable")
|
|
295
|
+
|
|
296
|
+
async def run(self, kind: str, payload: dict) -> Response:
|
|
297
|
+
if kind not in _QUERY_KINDS:
|
|
298
|
+
raise ValueError("unknown query job")
|
|
299
|
+
if self._closed or len(self._query_tasks) >= QUERY_SLOTS:
|
|
300
|
+
return _unavailable("work capacity exceeded")
|
|
301
|
+
task = asyncio.create_task(
|
|
302
|
+
self._run(kind, _request_data(kind, payload), mirror=False)
|
|
303
|
+
)
|
|
304
|
+
self._query_tasks.add(task)
|
|
305
|
+
try:
|
|
306
|
+
return await task
|
|
307
|
+
finally:
|
|
308
|
+
self._query_tasks.discard(task)
|
|
309
|
+
|
|
310
|
+
def _submit_mirror(self, kind: str, payload: dict) -> asyncio.Task:
|
|
311
|
+
if kind not in _MIRROR_KINDS:
|
|
312
|
+
raise ValueError("unknown mirror job")
|
|
313
|
+
if self._closed or len(self._mirror_tasks) >= MIRROR_SLOTS + MIRROR_QUEUE_SIZE:
|
|
314
|
+
raise HTTPException(status_code=503, detail="mirror capacity exceeded")
|
|
315
|
+
request = _request_data(kind, payload)
|
|
316
|
+
path = Path(settings.mirror_path or ".").absolute()
|
|
317
|
+
while not path.exists():
|
|
318
|
+
path = path.parent
|
|
319
|
+
try:
|
|
320
|
+
free = shutil.disk_usage(path).free
|
|
321
|
+
except OSError:
|
|
322
|
+
free = 0
|
|
323
|
+
if free < MIRROR_FREE_BYTES:
|
|
324
|
+
logger.warning("mirror_storage_reserve_exhausted")
|
|
325
|
+
raise HTTPException(status_code=503, detail="mirror storage unavailable")
|
|
326
|
+
task = asyncio.create_task(self._run(kind, request, mirror=True))
|
|
327
|
+
self._mirror_tasks.add(task)
|
|
328
|
+
task.add_done_callback(self._mirror_tasks.discard)
|
|
329
|
+
return task
|
|
330
|
+
|
|
331
|
+
def submit_push(
|
|
332
|
+
self,
|
|
333
|
+
push: Push,
|
|
334
|
+
*,
|
|
335
|
+
fetch_repo: str | None = None,
|
|
336
|
+
fetch_clone_url: str | None = None,
|
|
337
|
+
) -> asyncio.Task:
|
|
338
|
+
key = (push.org_id, push.push_id)
|
|
339
|
+
active = self._push_tasks.get(key)
|
|
340
|
+
if active is not None and not active.done():
|
|
341
|
+
return active
|
|
342
|
+
task = self._submit_mirror(
|
|
343
|
+
"push",
|
|
344
|
+
{
|
|
345
|
+
"push": push.model_dump(mode="json"),
|
|
346
|
+
"fetch_repo": fetch_repo,
|
|
347
|
+
"fetch_clone_url": fetch_clone_url,
|
|
348
|
+
},
|
|
349
|
+
)
|
|
350
|
+
self._push_tasks[key] = task
|
|
351
|
+
|
|
352
|
+
def completed(done: asyncio.Task) -> None:
|
|
353
|
+
if self._push_tasks.get(key) is done:
|
|
354
|
+
self._push_tasks.pop(key)
|
|
355
|
+
|
|
356
|
+
task.add_done_callback(completed)
|
|
357
|
+
return task
|
|
358
|
+
|
|
359
|
+
async def rename(self, old_repo: str, new_repo: str) -> Response:
|
|
360
|
+
task = self._submit_mirror(
|
|
361
|
+
"rename", {"old_repo": old_repo, "new_repo": new_repo}
|
|
362
|
+
)
|
|
363
|
+
return await task
|
|
364
|
+
|
|
365
|
+
async def wait_background(self, task: asyncio.Task) -> None:
|
|
366
|
+
# A duplicate delivery observes the same work; its disconnect must not
|
|
367
|
+
# cancel work another accepted Push delivery also depends on.
|
|
368
|
+
response = await asyncio.shield(task)
|
|
369
|
+
if response.status_code != 200:
|
|
370
|
+
logger.warning(
|
|
371
|
+
"mirror_background_work_failed status=%s", response.status_code
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
async def close(self) -> None:
|
|
375
|
+
self._closed = True
|
|
376
|
+
tasks = self._query_tasks | self._mirror_tasks
|
|
377
|
+
for task in tasks:
|
|
378
|
+
task.cancel()
|
|
379
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
380
|
+
self._query_tasks.clear()
|
|
381
|
+
self._mirror_tasks.clear()
|
|
382
|
+
self._push_tasks.clear()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sediment-api
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Sediment ingest API: facts in through three doors
|
|
5
|
+
Project-URL: Source, https://github.com/sediment-ai/sediment
|
|
6
|
+
Project-URL: Documentation, https://github.com/sediment-ai/sediment/tree/main/docs
|
|
7
|
+
Project-URL: Issues, https://github.com/sediment-ai/sediment/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/sediment-ai/sediment/blob/main/CHANGELOG.md
|
|
9
|
+
License-Expression: AGPL-3.0-or-later
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Python: >=3.12
|
|
12
|
+
Requires-Dist: anyio>=4.15.1
|
|
13
|
+
Requires-Dist: fastapi>=0.141.1
|
|
14
|
+
Requires-Dist: httpx>=0.27
|
|
15
|
+
Requires-Dist: pydantic-settings>=2.15.0
|
|
16
|
+
Requires-Dist: sediment-capture==0.1.0
|
|
17
|
+
Requires-Dist: sediment-core==0.1.0
|
|
18
|
+
Requires-Dist: sediment-derive==0.1.0
|
|
19
|
+
Requires-Dist: sediment-export==0.1.0
|
|
20
|
+
Requires-Dist: uvicorn>=0.53.0
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
sediment_api/__init__.py,sha256=MTyRpFXjAntHsY96K8jFdK83b6VsH6hd-id2i2azGwA,246
|
|
2
|
+
sediment_api/config.py,sha256=800jolpeq8IvD2ymrS3_JP2q70BrOu1XBbcAjPw37yw,9695
|
|
3
|
+
sediment_api/database.py,sha256=GPlL2-fTIXsgMonKN5pvDTykJ095iGyqgfOsZ9BF3Pg,1339
|
|
4
|
+
sediment_api/deps.py,sha256=1H9SgCUYA3p53skHLuSdEIRs6eG8oFcbObXz4qlyfi4,7927
|
|
5
|
+
sediment_api/main.py,sha256=BsqtfhnN2E3crMI9CL5NLXtFMbZIB62X18nP4e5p1ng,7794
|
|
6
|
+
sediment_api/mirror_gc.py,sha256=6nQCrvIvNQunBCVRpbBe3QLuIENZd7Igf5LCU4V1j7E,5221
|
|
7
|
+
sediment_api/worker.py,sha256=r-vsfs70nfJjtzaiPyRHGc0Pss_H99RAmtzV3ku64KI,9080
|
|
8
|
+
sediment_api/workers.py,sha256=p5WLNTCyMxVg15XP1u4PwoVa1Tx6F8Olyx4KNFE82Is,14292
|
|
9
|
+
sediment_api/reports/__init__.py,sha256=wxhDG1zJgpolGx9frRLaHiFRtHX80SR4hrQLZkoV4xU,448
|
|
10
|
+
sediment_api/reports/abandonment_report.py,sha256=99pwCYzEoNSRV2RRVy00n8GsszTb-yFR5q5t7ri_h1k,4276
|
|
11
|
+
sediment_api/reports/attribution_share_report.py,sha256=E4lZ-qMitu-94uxvD_jMnOsAXgLCv2ehWgJQq2WWnZA,10583
|
|
12
|
+
sediment_api/reports/dataset_diagnostics.py,sha256=DGT4cQbckosTBNix9qzphwjWsG42ay2v0tVFlU1MCmc,10512
|
|
13
|
+
sediment_api/reports/label_confidence_inspection.py,sha256=orwn_-vK3883r7MoLUtCnxglLtlHg-sJc1Nk3GxczN0,15944
|
|
14
|
+
sediment_api/reports/lifecycle_report.py,sha256=pg9R99LHIqr4AaO87QktJ2Q_RXHCgm7GJziD88BjFZE,2147
|
|
15
|
+
sediment_api/reports/merge_retention_report.py,sha256=N0lU4IyV3nVJdBp6hKv3_fCW_vOlDpgxHj2wePuRya0,5894
|
|
16
|
+
sediment_api/reports/model_report.py,sha256=PJNCmA2hRlEUVl8qlL7UQo3ZNy-2JgotdeV9uzxYcQg,56515
|
|
17
|
+
sediment_api/reports/precision_report.py,sha256=EPdNuDzi3ZSG57ooXKLQKRyXCzOLZ3vuyGQ8rPwqfa0,6273
|
|
18
|
+
sediment_api/reports/recovery_yield_report.py,sha256=0cWgroXEK45-MfQQ04xCh5jsqAaE1jgOFnwvQXEVHBg,4145
|
|
19
|
+
sediment_api/routers/__init__.py,sha256=VT8JPPpzx3Yz5kaUCFVaAYguQZ2aogK3_-FvFUHJ7YM,169
|
|
20
|
+
sediment_api/routers/ci_vendor.py,sha256=kkJSB2QBwZ-Ei3W5Osw48mfV2B9Ggq2OtwGn3MKd7eY,5295
|
|
21
|
+
sediment_api/routers/forge.py,sha256=s-WpXjoT_h4ClouUcSkiCxnzba0EQ_FT-Z7_tYw_a2o,17281
|
|
22
|
+
sediment_api/routers/gateway.py,sha256=GJ5WfSbobdgbdCosTq16VCpkBMBKJu6P-tt3XxURs28,4997
|
|
23
|
+
sediment_api/routers/otlp.py,sha256=tnmOgS7VG-4vryJBEWR3EQ9Dw6IyK7hIvPtBUeWy220,4288
|
|
24
|
+
sediment_api/routers/query.py,sha256=Cy0ASeu8zr0BS-AkZXVH5iu6cYTeYgSfZHxlblIQX5c,35880
|
|
25
|
+
sediment_api/routers/reports.py,sha256=gT4MI1pV9OgxJdPLBFHam92fZ2fcsrdSOUb8AqIBMn4,3395
|
|
26
|
+
sediment_api/routers/v1.py,sha256=1Mi9ubUQRn5zDpcLa7hP1gstfyJhKGIUdAZuWtZP8bw,6147
|
|
27
|
+
sediment_api/services/__init__.py,sha256=adnJxvXtJxWpuxauCYUxaPywjPkS6f0m0h78opQmMjQ,111
|
|
28
|
+
sediment_api/services/operational_reports.py,sha256=WOJ8HAk2vJcrfVh2fw609-FABzqv6oCBGkslCf85QHE,14197
|
|
29
|
+
sediment_api-0.1.0.dist-info/METADATA,sha256=0i16X18i5g4iS0CkLpiObe2W6egk1uAv7tkHuUBfq_g,809
|
|
30
|
+
sediment_api-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
31
|
+
sediment_api-0.1.0.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
|
|
32
|
+
sediment_api-0.1.0.dist-info/RECORD,,
|