shadowset-runtime 0.1.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. shadowset/__init__.py +3 -0
  2. shadowset/__main__.py +6 -0
  3. shadowset/api/__init__.py +1 -0
  4. shadowset/api/app.py +95 -0
  5. shadowset/api/contract.py +80 -0
  6. shadowset/api/errors.py +327 -0
  7. shadowset/api/idempotency.py +297 -0
  8. shadowset/api/middleware.py +81 -0
  9. shadowset/api/models.py +394 -0
  10. shadowset/api/routes/__init__.py +1 -0
  11. shadowset/api/routes/artifacts.py +370 -0
  12. shadowset/api/routes/compile.py +128 -0
  13. shadowset/api/routes/expand.py +401 -0
  14. shadowset/api/routes/health.py +85 -0
  15. shadowset/api/routes/observations.py +296 -0
  16. shadowset/api/routes/representations.py +148 -0
  17. shadowset/api/routes/retrieve.py +169 -0
  18. shadowset/api/routes/runs.py +121 -0
  19. shadowset/api/services.py +198 -0
  20. shadowset/artifacts/__init__.py +1 -0
  21. shadowset/artifacts/filesystem.py +133 -0
  22. shadowset/artifacts/s3.py +21 -0
  23. shadowset/artifacts/store.py +97 -0
  24. shadowset/cli.py +185 -0
  25. shadowset/context/__init__.py +57 -0
  26. shadowset/context/budget.py +60 -0
  27. shadowset/context/compiler.py +360 -0
  28. shadowset/context/policy.py +111 -0
  29. shadowset/context/source.py +53 -0
  30. shadowset/context/store.py +59 -0
  31. shadowset/derivation/__init__.py +42 -0
  32. shadowset/derivation/generators.py +190 -0
  33. shadowset/derivation/queue.py +354 -0
  34. shadowset/derivation/status.py +54 -0
  35. shadowset/derivation/worker.py +289 -0
  36. shadowset/ids.py +47 -0
  37. shadowset/index/__init__.py +42 -0
  38. shadowset/index/candidates.py +112 -0
  39. shadowset/index/embeddings.py +49 -0
  40. shadowset/index/fts.py +262 -0
  41. shadowset/index/text.py +26 -0
  42. shadowset/lock.py +163 -0
  43. shadowset/logging.py +71 -0
  44. shadowset/metadata/__init__.py +51 -0
  45. shadowset/metadata/_time.py +19 -0
  46. shadowset/metadata/migrations/0001_initial.sql +77 -0
  47. shadowset/metadata/migrations/0002_derivation_jobs.sql +40 -0
  48. shadowset/metadata/migrations/0003_idempotency_keys.sql +28 -0
  49. shadowset/metadata/migrations/0004_index_entries.sql +32 -0
  50. shadowset/metadata/migrations/0005_context_compilations.sql +48 -0
  51. shadowset/metadata/migrations/__init__.py +135 -0
  52. shadowset/metadata/repository.py +936 -0
  53. shadowset/metadata/sqlite.py +120 -0
  54. shadowset/openapi.py +272 -0
  55. shadowset/settings.py +131 -0
  56. shadowset/telemetry.py +303 -0
  57. shadowset_runtime-0.1.0a0.dist-info/METADATA +190 -0
  58. shadowset_runtime-0.1.0a0.dist-info/RECORD +60 -0
  59. shadowset_runtime-0.1.0a0.dist-info/WHEEL +4 -0
  60. shadowset_runtime-0.1.0a0.dist-info/entry_points.txt +2 -0
shadowset/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """Shadowset agent memory runtime."""
2
+
3
+ __version__ = "0.1.0a0"
shadowset/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ import sys
2
+
3
+ from shadowset.cli import main
4
+
5
+ if __name__ == "__main__":
6
+ sys.exit(main())
@@ -0,0 +1 @@
1
+ """HTTP surface of the daemon: the ``/v1`` wire contract."""
shadowset/api/app.py ADDED
@@ -0,0 +1,95 @@
1
+ """FastAPI application factory for the Shadowset daemon."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from collections.abc import AsyncIterator
7
+ from contextlib import asynccontextmanager
8
+
9
+ from fastapi import FastAPI
10
+ from opentelemetry import trace
11
+
12
+ from shadowset import __version__
13
+ from shadowset.api.errors import install_error_handlers
14
+ from shadowset.api.middleware import RequestIdMiddleware
15
+ from shadowset.api.routes import (
16
+ artifacts,
17
+ compile,
18
+ expand,
19
+ health,
20
+ observations,
21
+ representations,
22
+ retrieve,
23
+ runs,
24
+ )
25
+ from shadowset.api.services import open_services
26
+ from shadowset.settings import Settings
27
+ from shadowset.telemetry import instrument_app, shutdown_telemetry
28
+
29
+ log = logging.getLogger("shadowset.daemon")
30
+
31
+
32
+ def create_app(
33
+ settings: Settings, *, tracer_provider: trace.TracerProvider | None = None
34
+ ) -> FastAPI:
35
+ """Build the daemon app over ``settings``.
36
+
37
+ Tests call this directly against a temporary data directory; ``shadowset serve``
38
+ calls it with settings resolved from CLI flags and the environment.
39
+ ``tracer_provider`` lets tests capture spans (request and derivation) without touching
40
+ the global provider.
41
+ """
42
+
43
+ @asynccontextmanager
44
+ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
45
+ services = open_services(settings, tracer_provider=tracer_provider)
46
+ app.state.services = services
47
+ log.info(
48
+ "shadowsetd starting",
49
+ extra={
50
+ "version": __version__,
51
+ "data_dir": str(settings.data_dir.resolve()),
52
+ "url": settings.base_url,
53
+ "derivation_worker": settings.derivation_worker,
54
+ },
55
+ )
56
+ try:
57
+ services.worker.start() # no-op when derivation_worker is "off"
58
+ yield
59
+ finally:
60
+ # Stop the worker before closing the store it writes to; an in-flight job that
61
+ # outlives the timeout keeps its lease, which the next daemon's sweep returns.
62
+ if services.worker.stop(settings.derivation_shutdown_timeout_seconds):
63
+ log.info("derivation worker stopped")
64
+ else:
65
+ log.warning("derivation worker did not stop within the shutdown timeout")
66
+ services.close()
67
+ log.info("shadowsetd stopped")
68
+ # Here, not atexit: uvicorn re-raises the stop signal after the lifespan,
69
+ # skipping the SDK's exit hooks, so queued spans would otherwise be dropped.
70
+ shutdown_telemetry()
71
+
72
+ app = FastAPI(
73
+ title="Shadowset",
74
+ version=__version__,
75
+ lifespan=lifespan,
76
+ docs_url=None,
77
+ redoc_url=None,
78
+ openapi_url=None,
79
+ )
80
+ app.state.settings = settings
81
+ app.include_router(health.router)
82
+ app.include_router(runs.router)
83
+ app.include_router(artifacts.router)
84
+ app.include_router(observations.router)
85
+ app.include_router(retrieve.router)
86
+ app.include_router(expand.router)
87
+ app.include_router(representations.router)
88
+ app.include_router(compile.router)
89
+ # Order matters: add_middleware wraps outermost-last. Required nesting, outermost
90
+ # first: OpenTelemetryMiddleware > RequestIdMiddleware > UnhandledErrorMiddleware,
91
+ # so the access log runs inside the server span and can carry its trace_id.
92
+ install_error_handlers(app)
93
+ app.add_middleware(RequestIdMiddleware)
94
+ instrument_app(app, settings, tracer_provider=tracer_provider)
95
+ return app
@@ -0,0 +1,80 @@
1
+ """Shared OpenAPI declarations for the /v1 routes.
2
+
3
+ Documentation only: nothing here changes how a request is handled. The headers declared
4
+ here are implemented by ``RequestIdMiddleware`` (``X-Request-Id``) and the OpenTelemetry
5
+ ASGI instrumentation (``traceparent`` / ``tracestate``); the error envelope is produced by
6
+ the handlers in ``shadowset.api.errors``. Routes reference these so the generated
7
+ ``openapi/shadowset-v1.yaml`` describes every header and error shape a client can see
8
+ without the exporter having to patch the document afterwards.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import Any
14
+
15
+ from shadowset.api.errors import ErrorEnvelope
16
+
17
+ REQUEST_ID_DESCRIPTION = (
18
+ "Correlation id for this request. A client may supply its own (alphanumeric start, "
19
+ "then letters, digits, '.', '_', ':' or '-', at most 128 characters); otherwise the "
20
+ "daemon generates a 'req_...' id. A supplied id outside that format is not rejected "
21
+ "but replaced by a generated one. The response echoes the id that was used."
22
+ )
23
+
24
+ TRACE_CONTEXT: dict[str, Any] = {
25
+ "parameters": [
26
+ {
27
+ "name": "X-Request-Id",
28
+ "in": "header",
29
+ "required": False,
30
+ "description": REQUEST_ID_DESCRIPTION,
31
+ # No ``pattern``: the schema would promise a 4xx for a malformed id, and the
32
+ # middleware substitutes a generated id instead (see ``RequestIdMiddleware``).
33
+ "schema": {"type": "string"},
34
+ },
35
+ {
36
+ "name": "traceparent",
37
+ "in": "header",
38
+ "required": False,
39
+ "description": "W3C Trace Context parent; the daemon's server span joins this trace.",
40
+ "schema": {"type": "string"},
41
+ },
42
+ {
43
+ "name": "tracestate",
44
+ "in": "header",
45
+ "required": False,
46
+ "description": "W3C Trace Context vendor state, forwarded alongside traceparent.",
47
+ "schema": {"type": "string"},
48
+ },
49
+ ]
50
+ }
51
+ """``openapi_extra`` for every route: the request headers the middleware understands."""
52
+
53
+ REQUEST_ID_HEADER: dict[str, Any] = {
54
+ "X-Request-Id": {
55
+ "description": "The request id that was used; equals the request header when one "
56
+ "was supplied and accepted.",
57
+ "schema": {"type": "string"},
58
+ }
59
+ }
60
+ """Response header present on every response, success or error."""
61
+
62
+
63
+ def described(description: str, **extra: Any) -> dict[str, Any]:
64
+ """An additional-response entry carrying ``X-Request-Id`` plus ``extra`` keys.
65
+
66
+ ``extra`` may add ``headers`` (merged with ``X-Request-Id``), ``content`` or ``model``.
67
+ """
68
+ headers = {**REQUEST_ID_HEADER, **extra.pop("headers", {})}
69
+ return {"description": description, "headers": headers, **extra}
70
+
71
+
72
+ def error_response(description: str) -> dict[str, Any]:
73
+ """An error response: the ``ErrorEnvelope`` body with the request id header."""
74
+ return described(description, model=ErrorEnvelope)
75
+
76
+
77
+ INTERNAL_ERROR_RESPONSE: dict[int | str, dict[str, Any]] = {
78
+ 500: error_response("Unexpected failure; code 'internal_error'.")
79
+ }
80
+ """Router-level default so every operation documents the last-resort 500."""
@@ -0,0 +1,327 @@
1
+ """Error envelope and stable error codes for the wire contract.
2
+
3
+ Every JSON error the daemon returns has one shape::
4
+
5
+ {"error": {"code": "...", "message": "...", "request_id": "req_...", "details": {}}}
6
+
7
+ Clients key on ``code``; messages are human-readable and may change.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import enum
13
+ import logging
14
+ from collections.abc import Iterable, Iterator, Mapping
15
+ from typing import Any
16
+
17
+ from fastapi import FastAPI, Request
18
+ from fastapi.exceptions import RequestValidationError
19
+ from fastapi.responses import JSONResponse
20
+ from pydantic import BaseModel
21
+ from starlette.exceptions import HTTPException
22
+ from starlette.routing import Match
23
+ from starlette.types import ASGIApp, Message, Receive, Scope, Send
24
+
25
+ error_log = logging.getLogger("shadowset.error")
26
+
27
+
28
+ class ErrorCode(enum.StrEnum):
29
+ """The closed set of machine-readable error codes."""
30
+
31
+ INVALID_REQUEST = "invalid_request"
32
+ RUN_NOT_FOUND = "run_not_found"
33
+ ARTIFACT_NOT_FOUND = "artifact_not_found"
34
+ ARTIFACT_HASH_MISMATCH = "artifact_hash_mismatch"
35
+ OBSERVATION_NOT_FOUND = "observation_not_found"
36
+ REPRESENTATION_NOT_FOUND = "representation_not_found"
37
+ DERIVATION_NOT_READY = "derivation_not_ready"
38
+ BUDGET_EXHAUSTED = "budget_exhausted"
39
+ INTERNAL_ERROR = "internal_error"
40
+
41
+
42
+ class ErrorBody(BaseModel):
43
+ code: ErrorCode
44
+ message: str
45
+ request_id: str
46
+ details: dict[str, Any]
47
+
48
+
49
+ class ErrorEnvelope(BaseModel):
50
+ error: ErrorBody
51
+
52
+
53
+ class ShadowsetError(Exception):
54
+ """Base for errors that map onto the wire envelope.
55
+
56
+ Subclasses pin ``code``, ``status_code`` and a default ``message``; callers may
57
+ override the message and attach structured ``details``.
58
+ """
59
+
60
+ code: ErrorCode
61
+ status_code: int
62
+ default_message: str
63
+
64
+ def __init__(self, message: str | None = None, *, details: dict[str, Any] | None = None):
65
+ self.message = message if message is not None else self.default_message
66
+ self.details: dict[str, Any] = dict(details) if details else {}
67
+ super().__init__(self.message)
68
+
69
+
70
+ class InvalidRequest(ShadowsetError):
71
+ """The request is malformed. Pydantic validation failures use 422 instead; this
72
+ 400 default is for hand-raised shape problems the validator cannot see."""
73
+
74
+ code = ErrorCode.INVALID_REQUEST
75
+ status_code = 400
76
+ default_message = "The request is malformed or missing required fields."
77
+
78
+
79
+ class IdempotencyKeyReused(ShadowsetError):
80
+ """An ``Idempotency-Key`` already stored for this endpoint arrived with a different body.
81
+
82
+ Same ``invalid_request`` code as other malformed requests, but 409 Conflict: the
83
+ request is well-formed on its own and conflicts with state the client created
84
+ earlier. ``details`` carries the header name, a fixed ``reason`` and the stored
85
+ key's ``created_at`` so the client can tell a stale key from a real reuse.
86
+ """
87
+
88
+ code = ErrorCode.INVALID_REQUEST
89
+ status_code = 409
90
+ default_message = "Idempotency-Key was already used with a different request body."
91
+
92
+ def __init__(self, *, created_at: str) -> None:
93
+ super().__init__(
94
+ details={
95
+ "header": "Idempotency-Key",
96
+ "reason": "key reused with a different request body",
97
+ "created_at": created_at,
98
+ }
99
+ )
100
+
101
+
102
+ class RunNotFound(ShadowsetError):
103
+ code = ErrorCode.RUN_NOT_FOUND
104
+ status_code = 404
105
+ default_message = "No run exists with the given id."
106
+
107
+
108
+ class ArtifactNotFound(ShadowsetError):
109
+ code = ErrorCode.ARTIFACT_NOT_FOUND
110
+ status_code = 404
111
+ default_message = "No artifact exists with the given digest."
112
+
113
+
114
+ class ArtifactHashMismatch(ShadowsetError):
115
+ code = ErrorCode.ARTIFACT_HASH_MISMATCH
116
+ status_code = 422
117
+ default_message = "Uploaded bytes do not match the requested SHA-256 digest."
118
+
119
+
120
+ class ObservationNotFound(ShadowsetError):
121
+ code = ErrorCode.OBSERVATION_NOT_FOUND
122
+ status_code = 404
123
+ default_message = "No observation exists with the given id."
124
+
125
+
126
+ class RepresentationNotFound(ShadowsetError):
127
+ code = ErrorCode.REPRESENTATION_NOT_FOUND
128
+ status_code = 404
129
+ default_message = "No representation exists with the given id."
130
+
131
+
132
+ class DerivationNotReady(ShadowsetError):
133
+ """The derivation exists but has not been produced yet.
134
+
135
+ Status is 409 Conflict rather than 202 Accepted: the request is well-formed and
136
+ nothing was accepted for processing, the resource is simply not in a state that
137
+ can satisfy it yet. Clients should retry later.
138
+ """
139
+
140
+ code = ErrorCode.DERIVATION_NOT_READY
141
+ status_code = 409
142
+ default_message = "The requested derivation has not been produced yet."
143
+
144
+
145
+ class BudgetExhausted(ShadowsetError):
146
+ code = ErrorCode.BUDGET_EXHAUSTED
147
+ status_code = 422
148
+ default_message = "The request cannot be satisfied within the given budget."
149
+
150
+
151
+ class InternalError(ShadowsetError):
152
+ code = ErrorCode.INTERNAL_ERROR
153
+ status_code = 500
154
+ default_message = "An internal error occurred."
155
+
156
+
157
+ FALLBACK_REQUEST_ID = "req_unknown"
158
+
159
+
160
+ def request_id_of(request: Request) -> str:
161
+ """The id ``RequestIdMiddleware`` assigned, or a fixed fallback if it did not run."""
162
+ return str(getattr(request.state, "request_id", FALLBACK_REQUEST_ID))
163
+
164
+
165
+ def envelope_response(
166
+ request: Request,
167
+ *,
168
+ status_code: int,
169
+ code: ErrorCode,
170
+ message: str,
171
+ details: dict[str, Any] | None = None,
172
+ headers: Mapping[str, str] | None = None,
173
+ ) -> JSONResponse:
174
+ envelope = ErrorEnvelope(
175
+ error=ErrorBody(
176
+ code=code,
177
+ message=message,
178
+ request_id=request_id_of(request),
179
+ details=details or {},
180
+ )
181
+ )
182
+ return JSONResponse(
183
+ status_code=status_code, content=envelope.model_dump(mode="json"), headers=headers
184
+ )
185
+
186
+
187
+ async def shadowset_error_handler(request: Request, exc: Exception) -> JSONResponse:
188
+ assert isinstance(exc, ShadowsetError)
189
+ return envelope_response(
190
+ request,
191
+ status_code=exc.status_code,
192
+ code=exc.code,
193
+ message=exc.message,
194
+ details=exc.details,
195
+ )
196
+
197
+
198
+ async def validation_error_handler(request: Request, exc: Exception) -> JSONResponse:
199
+ """Pydantic rejected the request (including a body that is not valid JSON)."""
200
+ assert isinstance(exc, RequestValidationError)
201
+ errors = [
202
+ {"loc": list(error["loc"]), "msg": error["msg"], "type": error["type"]}
203
+ for error in exc.errors()
204
+ ]
205
+ return envelope_response(
206
+ request,
207
+ status_code=422,
208
+ code=ErrorCode.INVALID_REQUEST,
209
+ message=InvalidRequest.default_message,
210
+ details={"errors": errors},
211
+ )
212
+
213
+
214
+ async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
215
+ """Starlette-level HTTP errors: routing 404/405 and hand-raised ``HTTPException``.
216
+
217
+ A routing 404 is ``invalid_request`` (the path is not one the daemon serves);
218
+ resource-specific ``*_not_found`` codes come from endpoints that exist.
219
+ """
220
+ assert isinstance(exc, HTTPException)
221
+ code = ErrorCode.INTERNAL_ERROR if exc.status_code >= 500 else ErrorCode.INVALID_REQUEST
222
+ headers = dict(exc.headers or {})
223
+ if exc.status_code == 405 and (allowed := allowed_methods(request)):
224
+ headers["Allow"] = ", ".join(allowed)
225
+ return envelope_response(
226
+ request,
227
+ status_code=exc.status_code,
228
+ code=code,
229
+ message=str(exc.detail),
230
+ headers=headers,
231
+ )
232
+
233
+
234
+ def allowed_methods(request: Request) -> list[str]:
235
+ """Every method some route serves at the request's path, sorted.
236
+
237
+ Starlette's own 405 lists the methods of the first route whose path matched, and the
238
+ artifact operations are three routes on one path, so its ``Allow`` would name ``HEAD``
239
+ alone. RFC 9110 requires the full set.
240
+ """
241
+ methods: set[str] = set()
242
+ for route in _leaf_routes(request.app.routes):
243
+ match, _ = route.matches(request.scope)
244
+ if match is Match.PARTIAL:
245
+ methods |= set(getattr(route, "methods", None) or ())
246
+ return sorted(methods)
247
+
248
+
249
+ def _leaf_routes(routes: Iterable[Any]) -> Iterator[Any]:
250
+ """The operation routes behind ``include_router``.
251
+
252
+ FastAPI nests an included router's routes under a wrapper that exposes them as
253
+ ``original_router.routes``; a Starlette ``Mount`` or ``Router`` exposes ``routes``.
254
+ """
255
+ for route in routes:
256
+ router = getattr(route, "original_router", route)
257
+ nested = getattr(router, "routes", None)
258
+ if nested is not None:
259
+ yield from _leaf_routes(nested)
260
+ else:
261
+ yield route
262
+
263
+
264
+ class UnhandledErrorMiddleware:
265
+ """Last resort: turn any exception into a generic 500 envelope, logged exactly once.
266
+
267
+ This is a pure ASGI middleware rather than an ``app.add_exception_handler(Exception)``
268
+ handler on purpose. Starlette runs the latter in ``ServerErrorMiddleware``, the
269
+ outermost layer, which re-raises the exception afterwards so the server logs it a
270
+ second time (without the request id) and so ``RequestIdMiddleware`` never sees the
271
+ response. Catching here, inside ``RequestIdMiddleware``, means the exception stops
272
+ at this layer: one ERROR record on ``shadowset.error`` with the traceback and
273
+ request id, and the 500 envelope gets its ``X-Request-Id`` header and access-log
274
+ line like every other response.
275
+
276
+ The traceback never reaches the client. If the response has already started when
277
+ the exception fires, nothing coherent can be sent, so it is re-raised instead.
278
+ """
279
+
280
+ def __init__(self, app: ASGIApp) -> None:
281
+ self.app = app
282
+
283
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
284
+ if scope["type"] != "http":
285
+ await self.app(scope, receive, send)
286
+ return
287
+
288
+ response_started = False
289
+
290
+ async def send_tracking_start(message: Message) -> None:
291
+ nonlocal response_started
292
+ if message["type"] == "http.response.start":
293
+ response_started = True
294
+ await send(message)
295
+
296
+ try:
297
+ await self.app(scope, receive, send_tracking_start)
298
+ except Exception as exc:
299
+ if response_started:
300
+ raise
301
+ request = Request(scope, receive)
302
+ error_log.error(
303
+ "unhandled exception while serving %s %s",
304
+ request.method,
305
+ request.url.path,
306
+ exc_info=exc,
307
+ extra={"request_id": request_id_of(request)},
308
+ )
309
+ response = envelope_response(
310
+ request,
311
+ status_code=InternalError.status_code,
312
+ code=ErrorCode.INTERNAL_ERROR,
313
+ message=InternalError.default_message,
314
+ )
315
+ await response(scope, receive, send)
316
+
317
+
318
+ def install_error_handlers(app: FastAPI) -> None:
319
+ """Make every error response from ``app`` use the ``ErrorEnvelope`` shape.
320
+
321
+ Must run before ``RequestIdMiddleware`` is added: ``add_middleware`` wraps
322
+ outermost-last, and ``UnhandledErrorMiddleware`` has to sit inside it.
323
+ """
324
+ app.add_exception_handler(ShadowsetError, shadowset_error_handler)
325
+ app.add_exception_handler(RequestValidationError, validation_error_handler)
326
+ app.add_exception_handler(HTTPException, http_exception_handler)
327
+ app.add_middleware(UnhandledErrorMiddleware)