sediment-api 0.6.1__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sediment_api-0.6.1 → sediment_api-0.7.1}/PKG-INFO +7 -7
- {sediment_api-0.6.1 → sediment_api-0.7.1}/pyproject.toml +7 -7
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/__init__.py +1 -1
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/deps.py +1 -2
- sediment_api-0.7.1/sediment_api/ingest_envelopes.py +55 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/main.py +2 -1
- sediment_api-0.7.1/sediment_api/reports/compute_report.py +157 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/ci_vendor.py +3 -4
- sediment_api-0.7.1/sediment_api/routers/compute.py +101 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/forge.py +21 -22
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/gateway.py +4 -3
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/otlp.py +3 -3
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/query.py +43 -1
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/v1.py +1 -0
- sediment_api-0.7.1/tests/test_compute_ingest.py +124 -0
- sediment_api-0.7.1/tests/test_compute_report_cli.py +112 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_credential_authorities.py +1 -1
- sediment_api-0.7.1/tests/test_ingest_envelopes.py +139 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_v1.py +2 -1
- {sediment_api-0.6.1 → sediment_api-0.7.1}/.gitignore +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/LICENSE +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/config.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/database.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/mirror_gc.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/__init__.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/abandonment_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/attribution_share_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/dataset_diagnostics.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/label_confidence_inspection.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/lifecycle_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/merge_retention_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/model_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/precision_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/reports/recovery_yield_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/__init__.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/routers/reports.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/services/__init__.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/services/operational_reports.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/worker.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/sediment_api/workers.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/conftest.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_attribution_share_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_baseline_gap_repro.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_bug_repro_offset.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_capture_record_isolation.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_ci_vendor.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_config.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_context_discovery_authority.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_context_discovery_query.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_context_discovery_workers.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_context_evidence_query.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_context_query.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_database_lifecycle.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_database_outage.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_database_role_startup.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_dataset_diagnostics_report.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_e2e_smoke.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_evidence_query.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_evidence_worker_imports.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_evidence_workers.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_gateway_capture_receipt.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_gateway_identity.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_lifecycle_report_cli.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_merge_retention_cli.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_one_shot_database.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_operational_reports.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_operational_reports_service.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_postgres_gateway_tracer.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_push_mirror.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_query.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_report_attachment_scope.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_report_routes.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_repository_identity_ingest.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_repository_identity_model_service.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_repository_identity_queries.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_retrieval_authority.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_worker_processes.py +0 -0
- {sediment_api-0.6.1 → sediment_api-0.7.1}/tests/test_worker_routes.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sediment-api
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Sediment ingest API: facts in through three doors
|
|
5
5
|
Project-URL: Source, https://github.com/sediment-ai/sediment
|
|
6
6
|
Project-URL: Documentation, https://github.com/sediment-ai/sediment/tree/main/docs
|
|
@@ -9,12 +9,12 @@ Project-URL: Changelog, https://github.com/sediment-ai/sediment/blob/main/CHANGE
|
|
|
9
9
|
License-Expression: AGPL-3.0-or-later
|
|
10
10
|
License-File: LICENSE
|
|
11
11
|
Requires-Python: >=3.12
|
|
12
|
-
Requires-Dist: anyio>=4.
|
|
12
|
+
Requires-Dist: anyio>=4.16.0
|
|
13
13
|
Requires-Dist: fastapi>=0.142.2
|
|
14
14
|
Requires-Dist: httpx>=0.27
|
|
15
|
-
Requires-Dist: pydantic-settings>=2.16.
|
|
16
|
-
Requires-Dist: sediment-capture==0.
|
|
17
|
-
Requires-Dist: sediment-core==0.
|
|
18
|
-
Requires-Dist: sediment-derive==0.
|
|
19
|
-
Requires-Dist: sediment-export==0.
|
|
15
|
+
Requires-Dist: pydantic-settings>=2.16.2
|
|
16
|
+
Requires-Dist: sediment-capture==0.7.1
|
|
17
|
+
Requires-Dist: sediment-core==0.7.1
|
|
18
|
+
Requires-Dist: sediment-derive==0.7.1
|
|
19
|
+
Requires-Dist: sediment-export==0.7.1
|
|
20
20
|
Requires-Dist: uvicorn>=0.54.0
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sediment-api"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.1"
|
|
4
4
|
description = "Sediment ingest API: facts in through three doors"
|
|
5
5
|
requires-python = ">=3.12"
|
|
6
6
|
license = "AGPL-3.0-or-later"
|
|
7
7
|
license-files = ["LICENSE"]
|
|
8
8
|
dependencies = [
|
|
9
9
|
# Security floor for HTTPX/Starlette, including installations from wheels.
|
|
10
|
-
"anyio>=4.
|
|
10
|
+
"anyio>=4.16.0",
|
|
11
11
|
"fastapi>=0.142.2",
|
|
12
12
|
"httpx>=0.27",
|
|
13
13
|
"uvicorn>=0.54.0",
|
|
14
|
-
"pydantic-settings>=2.16.
|
|
15
|
-
"sediment-core==0.
|
|
16
|
-
"sediment-capture==0.
|
|
17
|
-
"sediment-derive==0.
|
|
18
|
-
"sediment-export==0.
|
|
14
|
+
"pydantic-settings>=2.16.2",
|
|
15
|
+
"sediment-core==0.7.1",
|
|
16
|
+
"sediment-capture==0.7.1",
|
|
17
|
+
"sediment-derive==0.7.1",
|
|
18
|
+
"sediment-export==0.7.1",
|
|
19
19
|
]
|
|
20
20
|
|
|
21
21
|
[project.urls]
|
|
@@ -14,8 +14,7 @@ from __future__ import annotations
|
|
|
14
14
|
import json
|
|
15
15
|
import secrets
|
|
16
16
|
from dataclasses import dataclass
|
|
17
|
-
from typing import Literal
|
|
18
|
-
from typing import Any
|
|
17
|
+
from typing import Any, Literal
|
|
19
18
|
|
|
20
19
|
from fastapi import Depends, Header, HTTPException, Request
|
|
21
20
|
from sediment_capture import verify_signature
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Typed response envelopes for the ingest doors (AGENTS.md §API Conventions).
|
|
3
|
+
|
|
4
|
+
Routers return these instead of bare dicts so the generated OpenAPI names
|
|
5
|
+
each outcome. The wire bodies stay exactly the documented shapes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Annotated, Literal
|
|
12
|
+
|
|
13
|
+
from pydantic import ConfigDict, Field
|
|
14
|
+
|
|
15
|
+
_CLOSED = ConfigDict(extra="forbid")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class IngestReceipt:
|
|
20
|
+
"""The retained Fact for one single-Fact ingest request."""
|
|
21
|
+
|
|
22
|
+
__pydantic_config__ = _CLOSED
|
|
23
|
+
|
|
24
|
+
fact_id: Annotated[
|
|
25
|
+
str,
|
|
26
|
+
Field(description="ID of the retained Fact, including on redelivery"),
|
|
27
|
+
]
|
|
28
|
+
stored: Annotated[
|
|
29
|
+
bool,
|
|
30
|
+
Field(
|
|
31
|
+
description=(
|
|
32
|
+
"False when a redelivery collapsed on a UNIQUE index (ADR "
|
|
33
|
+
"0003); success, not an error"
|
|
34
|
+
)
|
|
35
|
+
),
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class IngestSkipped:
|
|
41
|
+
"""A declined delivery that the sender shouldn't retry."""
|
|
42
|
+
|
|
43
|
+
__pydantic_config__ = _CLOSED
|
|
44
|
+
|
|
45
|
+
skipped: Literal[True]
|
|
46
|
+
reason: Annotated[
|
|
47
|
+
str, Field(description="Why the route stored nothing for this delivery")
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class OTLPExportLogsResponse:
|
|
53
|
+
"""The empty OTLP/HTTP JSON ExportLogsServiceResponse: full success."""
|
|
54
|
+
|
|
55
|
+
__pydantic_config__ = _CLOSED
|
|
@@ -34,7 +34,7 @@ from sqlalchemy.exc import (
|
|
|
34
34
|
from . import __version__
|
|
35
35
|
from .config import settings
|
|
36
36
|
from .deps import BodySizeLimitMiddleware
|
|
37
|
-
from .routers import ci_vendor, forge, gateway, otlp, query, reports, v1
|
|
37
|
+
from .routers import ci_vendor, compute, forge, gateway, otlp, query, reports, v1
|
|
38
38
|
from .workers import WorkerSupervisor
|
|
39
39
|
|
|
40
40
|
logging.basicConfig(
|
|
@@ -229,6 +229,7 @@ app.include_router(gateway.router, prefix="/ingest")
|
|
|
229
229
|
# free for a vendor-neutral push door later.
|
|
230
230
|
app.include_router(forge.router, prefix="/ingest/github")
|
|
231
231
|
app.include_router(ci_vendor.router, prefix="/ingest")
|
|
232
|
+
app.include_router(compute.router, prefix="/ingest")
|
|
232
233
|
# OTLP receiver: path fixed by the exporter (<endpoint>/v1/logs), no prefix.
|
|
233
234
|
app.include_router(otlp.router)
|
|
234
235
|
app.include_router(query.router, prefix="/query")
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Compute report: Session completion time by final Compute allocation.
|
|
3
|
+
|
|
4
|
+
sediment report compute --org acme-corp
|
|
5
|
+
sediment report compute --org acme-corp --json
|
|
6
|
+
sediment report compute --org acme-corp --rows-out sessions.jsonl
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
from sediment_core import normalize_org_id
|
|
17
|
+
from sediment_derive import ComputeArm
|
|
18
|
+
from sediment_export import (
|
|
19
|
+
ComputeReport,
|
|
20
|
+
DurationDistribution,
|
|
21
|
+
compute_report_to_json,
|
|
22
|
+
compute_timings_to_export_rows,
|
|
23
|
+
generate_compute_report_result,
|
|
24
|
+
write_jsonl,
|
|
25
|
+
)
|
|
26
|
+
from sediment_export.significance import MedianDifference
|
|
27
|
+
|
|
28
|
+
from ..database import add_database_url_argument, one_shot_fact_store
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _arm(arm: ComputeArm) -> str:
|
|
32
|
+
cpu = "unknown" if arm.cpu_millicores is None else f"{arm.cpu_millicores}m"
|
|
33
|
+
memory = (
|
|
34
|
+
"unknown" if arm.memory_bytes is None else f"{arm.memory_bytes / 2**30:.2f}GiB"
|
|
35
|
+
)
|
|
36
|
+
return f"{arm.allocation_source.value} cpu={cpu} memory={memory}"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _seconds(milliseconds: float | None) -> str:
|
|
40
|
+
return "n/a" if milliseconds is None else f"{milliseconds / 1000:.1f}s"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _distribution(name: str, distribution: DurationDistribution) -> str:
|
|
44
|
+
return (
|
|
45
|
+
f"{name} median={_seconds(distribution.median)} "
|
|
46
|
+
f"p25={_seconds(distribution.p25)} p75={_seconds(distribution.p75)}"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _difference(name: str, difference: MedianDifference) -> str:
|
|
51
|
+
sizes = f"n={difference.n_a}/{difference.n_b}"
|
|
52
|
+
if difference.insufficient_data:
|
|
53
|
+
return f" {name}: insufficient data ({sizes})"
|
|
54
|
+
flag = " small-n" if difference.small_n else ""
|
|
55
|
+
level = round(difference.confidence * 100)
|
|
56
|
+
return (
|
|
57
|
+
f" {name}: {_seconds(difference.difference)} "
|
|
58
|
+
f"[{level}% CI {_seconds(difference.ci_low)}, "
|
|
59
|
+
f"{_seconds(difference.ci_high)}] {sizes}{flag}"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _print_table(report: ComputeReport) -> None:
|
|
64
|
+
print(f"Sessions with a Compute allocation: {report.sessions_with_allocation}")
|
|
65
|
+
print(f"timed Sessions: {report.timed_sessions}")
|
|
66
|
+
skipped = ", ".join(f"{key}={value}" for key, value in report.skipped.items())
|
|
67
|
+
print(f"skipped Sessions: {skipped}")
|
|
68
|
+
print(
|
|
69
|
+
"randomized Sessions reassigned to another source: "
|
|
70
|
+
f"{report.randomized_reassigned_sessions} "
|
|
71
|
+
"(excluded from randomized comparisons)"
|
|
72
|
+
)
|
|
73
|
+
if not report.arms:
|
|
74
|
+
print("arms: no data")
|
|
75
|
+
for summary in report.arms:
|
|
76
|
+
print(
|
|
77
|
+
f"{_arm(summary.arm)}: n={summary.sessions} "
|
|
78
|
+
f"(allocation changed in {summary.allocation_changed_sessions})"
|
|
79
|
+
)
|
|
80
|
+
for name in ("elapsed_ms", "model_ms", "between_calls_ms"):
|
|
81
|
+
print(f" {_distribution(name, getattr(summary, name))}")
|
|
82
|
+
if report.comparisons:
|
|
83
|
+
print("comparisons (arm B minus arm A; negative means arm B finished faster):")
|
|
84
|
+
for comparison in report.comparisons:
|
|
85
|
+
reading = "randomized" if comparison.randomized else "observational"
|
|
86
|
+
print(f" {_arm(comparison.arm_a)} -> {_arm(comparison.arm_b)} ({reading})")
|
|
87
|
+
print(_difference("elapsed_ms", comparison.elapsed_ms))
|
|
88
|
+
print(_difference("between_calls_ms", comparison.between_calls_ms))
|
|
89
|
+
print(f"provenance: {report.provenance}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
93
|
+
"""Return the argv contract consumed by the generated CLI reference."""
|
|
94
|
+
parser = argparse.ArgumentParser(
|
|
95
|
+
prog="compute_report",
|
|
96
|
+
description=(
|
|
97
|
+
"Compare Session completion time across final Compute allocations."
|
|
98
|
+
),
|
|
99
|
+
)
|
|
100
|
+
parser.add_argument(
|
|
101
|
+
"--org",
|
|
102
|
+
default=os.environ.get("SEDIMENT_ORG_ID"),
|
|
103
|
+
required=not os.environ.get("SEDIMENT_ORG_ID"),
|
|
104
|
+
help="org id (default: $SEDIMENT_ORG_ID)",
|
|
105
|
+
)
|
|
106
|
+
parser.add_argument(
|
|
107
|
+
"--json", action="store_true", help="emit JSON instead of a text report"
|
|
108
|
+
)
|
|
109
|
+
parser.add_argument(
|
|
110
|
+
"--rows-out",
|
|
111
|
+
metavar="FILE",
|
|
112
|
+
help="write one Session compute timing per line to FILE as JSONL",
|
|
113
|
+
)
|
|
114
|
+
add_database_url_argument(parser)
|
|
115
|
+
return parser
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def main(argv: list[str] | None = None) -> int:
|
|
119
|
+
args = build_parser().parse_args(argv)
|
|
120
|
+
try:
|
|
121
|
+
org_id = normalize_org_id(args.org)
|
|
122
|
+
except ValueError as exc:
|
|
123
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
124
|
+
return 2
|
|
125
|
+
with one_shot_fact_store(
|
|
126
|
+
args.database_url, operation="generate compute report"
|
|
127
|
+
) as store:
|
|
128
|
+
result = generate_compute_report_result(store, org_id)
|
|
129
|
+
if args.json:
|
|
130
|
+
print(json.dumps(compute_report_to_json(result.report), indent=2))
|
|
131
|
+
else:
|
|
132
|
+
_print_table(result.report)
|
|
133
|
+
if args.rows_out:
|
|
134
|
+
try:
|
|
135
|
+
write_result = write_jsonl(
|
|
136
|
+
compute_timings_to_export_rows(result.timings),
|
|
137
|
+
args.rows_out,
|
|
138
|
+
split_enabled=False,
|
|
139
|
+
)
|
|
140
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
141
|
+
print(f"error: couldn't write compute timing rows: {exc}", file=sys.stderr)
|
|
142
|
+
return 1
|
|
143
|
+
if write_result.written:
|
|
144
|
+
print(
|
|
145
|
+
f"wrote {len(result.timings)} compute timing rows to {args.rows_out}",
|
|
146
|
+
file=sys.stderr,
|
|
147
|
+
)
|
|
148
|
+
else:
|
|
149
|
+
print(
|
|
150
|
+
f"wrote 0 compute timing rows; left {args.rows_out} unchanged",
|
|
151
|
+
file=sys.stderr,
|
|
152
|
+
)
|
|
153
|
+
return 0
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
if __name__ == "__main__":
|
|
157
|
+
raise SystemExit(main())
|
|
@@ -16,8 +16,6 @@ collapses redeliveries; ``stored: false`` means success, not an error.
|
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
-
from typing import Any
|
|
20
|
-
|
|
21
19
|
from fastapi import APIRouter, Depends
|
|
22
20
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
23
21
|
from sediment_core import (
|
|
@@ -40,6 +38,7 @@ from sediment_core import (
|
|
|
40
38
|
|
|
41
39
|
from ..config import settings
|
|
42
40
|
from ..deps import get_store, verify_ingest_token
|
|
41
|
+
from ..ingest_envelopes import IngestReceipt
|
|
43
42
|
|
|
44
43
|
router = APIRouter(tags=["ci-vendor"])
|
|
45
44
|
|
|
@@ -121,7 +120,7 @@ def ingest_ci_vendor(
|
|
|
121
120
|
body: VendorCIRequest,
|
|
122
121
|
_: None = Depends(verify_ingest_token),
|
|
123
122
|
store: FactStore = Depends(get_store),
|
|
124
|
-
) ->
|
|
123
|
+
) -> IngestReceipt:
|
|
125
124
|
outcome = CIOutcome(
|
|
126
125
|
org_id=settings.org_id,
|
|
127
126
|
provider=body.provider,
|
|
@@ -146,4 +145,4 @@ def ingest_ci_vendor(
|
|
|
146
145
|
source_event_id=body.source_event_id,
|
|
147
146
|
)
|
|
148
147
|
receipt = store.store_ci_outcome_receipt(outcome)
|
|
149
|
-
return
|
|
148
|
+
return IngestReceipt(fact_id=receipt.fact_id, stored=receipt.stored)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""
|
|
3
|
+
POST /ingest/compute — vendor-neutral Compute allocation ingest.
|
|
4
|
+
|
|
5
|
+
The execution platform that runs a coding agent (a sandbox, a container
|
|
6
|
+
scheduler, a CI runner) posts the CPU and memory limits a Session's
|
|
7
|
+
environment received: once when the environment starts, and again on every
|
|
8
|
+
resize. ``allocation_source`` records who chose the allocation, so the
|
|
9
|
+
compute report never presents an agent's own request as an assigned
|
|
10
|
+
treatment. The sender is bearer-authenticated; Sediment stores its assertion
|
|
11
|
+
and does not verify it against the platform.
|
|
12
|
+
|
|
13
|
+
Dedup: the store's ``uq_compute_allocations_natural`` UNIQUE index on the
|
|
14
|
+
complete allocation collapses redeliveries; ``stored: false`` means success,
|
|
15
|
+
not an error.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from fastapi import APIRouter, Depends
|
|
21
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
22
|
+
from sediment_core import (
|
|
23
|
+
ComputeAllocation,
|
|
24
|
+
ComputeAllocationSource,
|
|
25
|
+
FactStore,
|
|
26
|
+
NonEmptyId,
|
|
27
|
+
UtcRepresentableDatetime,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
from ..config import settings
|
|
31
|
+
from ..deps import get_store, verify_ingest_token
|
|
32
|
+
from ..ingest_envelopes import IngestReceipt
|
|
33
|
+
|
|
34
|
+
router = APIRouter(tags=["compute"])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ComputeAllocationRequest(BaseModel):
|
|
38
|
+
"""One Compute allocation as the execution platform observed it.
|
|
39
|
+
|
|
40
|
+
``extra="forbid"`` so a caller naming an org or fabricating fields is
|
|
41
|
+
rejected at the boundary. The limits carry the Fact's own bounds, so
|
|
42
|
+
malformed input 422s at the door rather than inside the handler."""
|
|
43
|
+
|
|
44
|
+
model_config = ConfigDict(extra="forbid")
|
|
45
|
+
|
|
46
|
+
session_id: NonEmptyId = Field(
|
|
47
|
+
description=(
|
|
48
|
+
"The Session ID the coding agent's captured Facts carry, such as "
|
|
49
|
+
"the gateway Session header"
|
|
50
|
+
)
|
|
51
|
+
)
|
|
52
|
+
allocation_source: ComputeAllocationSource = Field(
|
|
53
|
+
description=(
|
|
54
|
+
"Who chose the allocation: default, agent_request, operator, or "
|
|
55
|
+
"randomized. Only randomized supports a causal comparison"
|
|
56
|
+
)
|
|
57
|
+
)
|
|
58
|
+
cpu_millicores: int | None = Field(
|
|
59
|
+
default=None,
|
|
60
|
+
ge=1,
|
|
61
|
+
le=2**63 - 1,
|
|
62
|
+
strict=True,
|
|
63
|
+
description="CPU limit in millicores; 1000 is one CPU",
|
|
64
|
+
)
|
|
65
|
+
memory_bytes: int | None = Field(
|
|
66
|
+
default=None,
|
|
67
|
+
ge=1,
|
|
68
|
+
le=2**63 - 1,
|
|
69
|
+
strict=True,
|
|
70
|
+
description="Memory limit in bytes",
|
|
71
|
+
)
|
|
72
|
+
occurred_at: UtcRepresentableDatetime = Field(
|
|
73
|
+
description=(
|
|
74
|
+
"When the allocation took effect, by the sender's clock; it must "
|
|
75
|
+
"convert to a UTC instant"
|
|
76
|
+
)
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
@model_validator(mode="after")
|
|
80
|
+
def _require_limit(self):
|
|
81
|
+
if self.cpu_millicores is None and self.memory_bytes is None:
|
|
82
|
+
raise ValueError("send cpu_millicores, memory_bytes, or both")
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@router.post("/compute")
|
|
87
|
+
def ingest_compute_allocation(
|
|
88
|
+
body: ComputeAllocationRequest,
|
|
89
|
+
_: None = Depends(verify_ingest_token),
|
|
90
|
+
store: FactStore = Depends(get_store),
|
|
91
|
+
) -> IngestReceipt:
|
|
92
|
+
allocation = ComputeAllocation(
|
|
93
|
+
org_id=settings.org_id,
|
|
94
|
+
session_id=body.session_id,
|
|
95
|
+
allocation_source=body.allocation_source,
|
|
96
|
+
cpu_millicores=body.cpu_millicores,
|
|
97
|
+
memory_bytes=body.memory_bytes,
|
|
98
|
+
occurred_at=body.occurred_at,
|
|
99
|
+
)
|
|
100
|
+
receipt = store.store_compute_allocation_receipt(allocation)
|
|
101
|
+
return IngestReceipt(fact_id=receipt.fact_id, stored=receipt.stored)
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
"""
|
|
3
3
|
GitHub webhook ingest routes, mounted under /ingest/github in main.py.
|
|
4
4
|
|
|
5
|
-
One module for
|
|
6
|
-
which parser/store pair the payload walks through. The signature is
|
|
5
|
+
One module for all four routes: they share the HMAC intake and differ only
|
|
6
|
+
in which parser/store pair the payload walks through. The signature is
|
|
7
7
|
verified BEFORE the event type is examined: a wrong-secret delivery must
|
|
8
8
|
401 even for event types a route skips, or GitHub's setup ping would
|
|
9
9
|
green-light a misconfigured webhook. The push route stores the Push fact,
|
|
@@ -19,7 +19,6 @@ from __future__ import annotations
|
|
|
19
19
|
import logging
|
|
20
20
|
from collections import Counter
|
|
21
21
|
from datetime import UTC, datetime
|
|
22
|
-
from typing import Any
|
|
23
22
|
|
|
24
23
|
from fastapi import APIRouter, BackgroundTasks, Depends, Header, Request
|
|
25
24
|
from sediment_capture import (
|
|
@@ -53,6 +52,7 @@ from starlette.concurrency import run_in_threadpool
|
|
|
53
52
|
|
|
54
53
|
from ..config import settings
|
|
55
54
|
from ..deps import get_store, read_verified_webhook
|
|
55
|
+
from ..ingest_envelopes import IngestReceipt, IngestSkipped
|
|
56
56
|
|
|
57
57
|
logger = logging.getLogger("sediment.api.forge")
|
|
58
58
|
router = APIRouter(tags=["forge"])
|
|
@@ -295,15 +295,15 @@ async def ingest_push(
|
|
|
295
295
|
x_hub_signature_256: str | None = Header(None),
|
|
296
296
|
x_github_event: str = Header(...),
|
|
297
297
|
store: FactStore = Depends(get_store),
|
|
298
|
-
) ->
|
|
298
|
+
) -> IngestReceipt | IngestSkipped:
|
|
299
299
|
payload = await read_verified_webhook(request, x_hub_signature_256)
|
|
300
300
|
# GitHub delivers every subscribed event (incl. ping) to the one URL;
|
|
301
301
|
# the X-GitHub-Event header is the discriminator.
|
|
302
302
|
if x_github_event != "push":
|
|
303
|
-
return
|
|
303
|
+
return IngestSkipped(skipped=True, reason="not a push event")
|
|
304
304
|
push = parse_push(payload, org_id=settings.org_id, github_host=settings.github_host)
|
|
305
305
|
if push is None:
|
|
306
|
-
return
|
|
306
|
+
return IngestSkipped(skipped=True, reason="not a storable push")
|
|
307
307
|
receipt = await run_in_threadpool(store.store_push_receipt, push)
|
|
308
308
|
fetch_repo, fetch_clone_url = push.repo, push.clone_url
|
|
309
309
|
push = receipt.fact
|
|
@@ -325,12 +325,11 @@ async def ingest_push(
|
|
|
325
325
|
push, fetch_repo=fetch_repo, fetch_clone_url=fetch_clone_url
|
|
326
326
|
)
|
|
327
327
|
background.add_task(request.app.state.workers.wait_background, task)
|
|
328
|
-
return
|
|
328
|
+
return IngestReceipt(fact_id=push.push_id, stored=stored)
|
|
329
329
|
|
|
330
330
|
|
|
331
331
|
@router.post(
|
|
332
332
|
"/repository",
|
|
333
|
-
response_model=dict[str, Any],
|
|
334
333
|
responses={
|
|
335
334
|
503: {
|
|
336
335
|
"description": "Mirror work capacity, storage, or execution budget exceeded."
|
|
@@ -343,11 +342,11 @@ async def ingest_repository(
|
|
|
343
342
|
x_github_event: str = Header(...),
|
|
344
343
|
x_github_delivery: str | None = Header(None),
|
|
345
344
|
store: FactStore = Depends(get_store),
|
|
346
|
-
) ->
|
|
345
|
+
) -> IngestReceipt | IngestSkipped:
|
|
347
346
|
"""Store an immutable rename receipt independently of mirror availability."""
|
|
348
347
|
payload = await read_verified_webhook(request, x_hub_signature_256)
|
|
349
348
|
if x_github_event != "repository":
|
|
350
|
-
return
|
|
349
|
+
return IngestSkipped(skipped=True, reason="not a repository event")
|
|
351
350
|
rename, reason = parse_repository_rename(
|
|
352
351
|
payload,
|
|
353
352
|
org_id=settings.org_id,
|
|
@@ -356,12 +355,12 @@ async def ingest_repository(
|
|
|
356
355
|
)
|
|
357
356
|
if rename is None:
|
|
358
357
|
assert reason is not None
|
|
359
|
-
return
|
|
358
|
+
return IngestSkipped(skipped=True, reason=reason.value)
|
|
360
359
|
receipt = await run_in_threadpool(store.store_repository_rename_receipt, rename)
|
|
361
360
|
logger.info(
|
|
362
361
|
"repository_rename_stored fact_id=%s stored=%s", receipt.fact_id, receipt.stored
|
|
363
362
|
)
|
|
364
|
-
return
|
|
363
|
+
return IngestReceipt(fact_id=receipt.fact_id, stored=receipt.stored)
|
|
365
364
|
|
|
366
365
|
|
|
367
366
|
@router.post("/ci")
|
|
@@ -371,10 +370,10 @@ async def ingest_ci(
|
|
|
371
370
|
x_github_event: str = Header(...),
|
|
372
371
|
x_github_delivery: str | None = Header(None),
|
|
373
372
|
store: FactStore = Depends(get_store),
|
|
374
|
-
) ->
|
|
373
|
+
) -> IngestReceipt | IngestSkipped:
|
|
375
374
|
payload = await read_verified_webhook(request, x_hub_signature_256)
|
|
376
375
|
if x_github_event != "workflow_run":
|
|
377
|
-
return
|
|
376
|
+
return IngestSkipped(skipped=True, reason="not a workflow_run event")
|
|
378
377
|
outcome = parse_workflow_run(
|
|
379
378
|
payload,
|
|
380
379
|
org_id=settings.org_id,
|
|
@@ -382,7 +381,7 @@ async def ingest_ci(
|
|
|
382
381
|
github_host=settings.github_host,
|
|
383
382
|
)
|
|
384
383
|
if outcome is None:
|
|
385
|
-
return
|
|
384
|
+
return IngestSkipped(skipped=True, reason="not a completed workflow_run")
|
|
386
385
|
receipt = await run_in_threadpool(store.store_ci_outcome_receipt, outcome)
|
|
387
386
|
outcome = receipt.fact
|
|
388
387
|
stored = receipt.stored
|
|
@@ -393,7 +392,7 @@ async def ingest_ci(
|
|
|
393
392
|
outcome.repo,
|
|
394
393
|
outcome.result,
|
|
395
394
|
)
|
|
396
|
-
return
|
|
395
|
+
return IngestReceipt(fact_id=outcome.outcome_id, stored=stored)
|
|
397
396
|
|
|
398
397
|
|
|
399
398
|
@router.post("/pull-request")
|
|
@@ -403,10 +402,10 @@ async def ingest_pull_request(
|
|
|
403
402
|
x_github_event: str = Header(...),
|
|
404
403
|
x_github_delivery: str | None = Header(None),
|
|
405
404
|
store: FactStore = Depends(get_store),
|
|
406
|
-
) ->
|
|
405
|
+
) -> IngestReceipt | IngestSkipped:
|
|
407
406
|
payload = await read_verified_webhook(request, x_hub_signature_256)
|
|
408
407
|
if x_github_event != "pull_request":
|
|
409
|
-
return
|
|
408
|
+
return IngestSkipped(skipped=True, reason="not a pull_request event")
|
|
410
409
|
if payload.get("action") == "closed":
|
|
411
410
|
merge = parse_pull_request_merge(
|
|
412
411
|
payload,
|
|
@@ -415,7 +414,7 @@ async def ingest_pull_request(
|
|
|
415
414
|
github_host=settings.github_host,
|
|
416
415
|
)
|
|
417
416
|
if merge is None:
|
|
418
|
-
return
|
|
417
|
+
return IngestSkipped(skipped=True, reason="not a merged pull_request")
|
|
419
418
|
receipt = await run_in_threadpool(store.store_pull_request_merge_receipt, merge)
|
|
420
419
|
merge = receipt.fact
|
|
421
420
|
stored = receipt.stored
|
|
@@ -426,7 +425,7 @@ async def ingest_pull_request(
|
|
|
426
425
|
merge.repo,
|
|
427
426
|
merge.pr_number,
|
|
428
427
|
)
|
|
429
|
-
return
|
|
428
|
+
return IngestReceipt(fact_id=merge.merge_id, stored=stored)
|
|
430
429
|
|
|
431
430
|
revision, skip_reason = parse_pull_request_revision(
|
|
432
431
|
payload,
|
|
@@ -436,7 +435,7 @@ async def ingest_pull_request(
|
|
|
436
435
|
)
|
|
437
436
|
if revision is None:
|
|
438
437
|
assert skip_reason is not None
|
|
439
|
-
return
|
|
438
|
+
return IngestSkipped(skipped=True, reason=skip_reason.value)
|
|
440
439
|
receipt = await run_in_threadpool(
|
|
441
440
|
store.store_pull_request_revision_receipt, revision
|
|
442
441
|
)
|
|
@@ -449,4 +448,4 @@ async def ingest_pull_request(
|
|
|
449
448
|
revision.repo,
|
|
450
449
|
revision.pr_number,
|
|
451
450
|
)
|
|
452
|
-
return
|
|
451
|
+
return IngestReceipt(fact_id=revision.revision_id, stored=stored)
|
|
@@ -20,6 +20,7 @@ from sediment_core.models import AwareDatetime
|
|
|
20
20
|
|
|
21
21
|
from ..config import settings
|
|
22
22
|
from ..deps import get_store, verify_ingest_token
|
|
23
|
+
from ..ingest_envelopes import IngestReceipt, IngestSkipped
|
|
23
24
|
|
|
24
25
|
logger = logging.getLogger("sediment.api.gateway")
|
|
25
26
|
router = APIRouter(tags=["gateway"])
|
|
@@ -57,7 +58,7 @@ def ingest_gateway(
|
|
|
57
58
|
body: GatewayIngestRequest,
|
|
58
59
|
_: None = Depends(verify_ingest_token),
|
|
59
60
|
store: FactStore = Depends(get_store),
|
|
60
|
-
) ->
|
|
61
|
+
) -> IngestReceipt | IngestSkipped:
|
|
61
62
|
adapter = ADAPTERS.get(body.provider)
|
|
62
63
|
if adapter is None:
|
|
63
64
|
raise HTTPException(
|
|
@@ -85,7 +86,7 @@ def ingest_gateway(
|
|
|
85
86
|
body.provider,
|
|
86
87
|
body.payload.get("litellm_call_id"),
|
|
87
88
|
)
|
|
88
|
-
return
|
|
89
|
+
return IngestSkipped(skipped=True, reason="no_session")
|
|
89
90
|
|
|
90
91
|
try:
|
|
91
92
|
inference_call = adapter.normalize(
|
|
@@ -134,4 +135,4 @@ def ingest_gateway(
|
|
|
134
135
|
receipt.fact_id,
|
|
135
136
|
receipt.stored,
|
|
136
137
|
)
|
|
137
|
-
return
|
|
138
|
+
return IngestReceipt(fact_id=receipt.fact_id, stored=receipt.stored)
|
|
@@ -12,7 +12,6 @@ from __future__ import annotations
|
|
|
12
12
|
|
|
13
13
|
import json
|
|
14
14
|
import logging
|
|
15
|
-
from typing import Any
|
|
16
15
|
|
|
17
16
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
18
17
|
from sediment_capture import parse_otlp_logs
|
|
@@ -21,6 +20,7 @@ from starlette.concurrency import run_in_threadpool
|
|
|
21
20
|
|
|
22
21
|
from ..config import settings
|
|
23
22
|
from ..deps import get_store, verify_ingest_token
|
|
23
|
+
from ..ingest_envelopes import OTLPExportLogsResponse
|
|
24
24
|
|
|
25
25
|
logger = logging.getLogger("sediment.api.otlp")
|
|
26
26
|
router = APIRouter(tags=["otlp"])
|
|
@@ -31,7 +31,7 @@ async def ingest_otlp_logs(
|
|
|
31
31
|
request: Request,
|
|
32
32
|
_: None = Depends(verify_ingest_token),
|
|
33
33
|
store: FactStore = Depends(get_store),
|
|
34
|
-
) ->
|
|
34
|
+
) -> OTLPExportLogsResponse:
|
|
35
35
|
try:
|
|
36
36
|
payload = await request.json()
|
|
37
37
|
except HTTPException:
|
|
@@ -98,7 +98,7 @@ async def ingest_otlp_logs(
|
|
|
98
98
|
},
|
|
99
99
|
)
|
|
100
100
|
# Empty ExportLogsServiceResponse = full success (OTLP/HTTP JSON).
|
|
101
|
-
return
|
|
101
|
+
return OTLPExportLogsResponse()
|
|
102
102
|
|
|
103
103
|
|
|
104
104
|
def _store_translated_facts(
|