memini 0.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (260) hide show
  1. memini/__init__.py +5 -0
  2. memini/__main__.py +6 -0
  3. memini/api/__init__.py +19 -0
  4. memini/api/admin_errors.py +68 -0
  5. memini/api/agent_runs.py +469 -0
  6. memini/api/app.py +276 -0
  7. memini/api/auth.py +192 -0
  8. memini/api/comms.py +191 -0
  9. memini/api/database.py +30 -0
  10. memini/api/decisions.py +140 -0
  11. memini/api/errors.py +224 -0
  12. memini/api/extract.py +178 -0
  13. memini/api/forget.py +156 -0
  14. memini/api/health.py +129 -0
  15. memini/api/ingest.py +469 -0
  16. memini/api/limits.py +503 -0
  17. memini/api/mcp.py +169 -0
  18. memini/api/metrics.py +422 -0
  19. memini/api/persons.py +373 -0
  20. memini/api/query.py +148 -0
  21. memini/api/request_id.py +79 -0
  22. memini/api/runs.py +119 -0
  23. memini/api/sources.py +978 -0
  24. memini/api/status.py +71 -0
  25. memini/api/tokens.py +158 -0
  26. memini/api/workspaces.py +79 -0
  27. memini/auth/__init__.py +72 -0
  28. memini/auth/authentication.py +156 -0
  29. memini/auth/authorization.py +205 -0
  30. memini/auth/principal.py +45 -0
  31. memini/auth/tokens.py +243 -0
  32. memini/bench/__init__.py +59 -0
  33. memini/bench/corpus.py +277 -0
  34. memini/bench/measure.py +124 -0
  35. memini/bench/queries.py +134 -0
  36. memini/bench/report.py +310 -0
  37. memini/bench/repository.py +213 -0
  38. memini/bench/runner.py +1086 -0
  39. memini/chunking/__init__.py +194 -0
  40. memini/chunking/chunker.py +584 -0
  41. memini/chunking/languages.py +414 -0
  42. memini/chunking/markdown.py +393 -0
  43. memini/chunking/model.py +213 -0
  44. memini/chunking/payloads.py +453 -0
  45. memini/chunking/text.py +163 -0
  46. memini/cli/__init__.py +597 -0
  47. memini/cli/admin.py +191 -0
  48. memini/cli/arguments.py +248 -0
  49. memini/cli/bench.py +321 -0
  50. memini/cli/client.py +493 -0
  51. memini/cli/common.py +221 -0
  52. memini/cli/comms.py +181 -0
  53. memini/cli/credentials.py +385 -0
  54. memini/cli/decisions.py +341 -0
  55. memini/cli/evaluate.py +193 -0
  56. memini/cli/evaluate_decisions.py +225 -0
  57. memini/cli/export.py +116 -0
  58. memini/cli/extract.py +102 -0
  59. memini/cli/forget.py +96 -0
  60. memini/cli/index.py +89 -0
  61. memini/cli/output.py +117 -0
  62. memini/cli/persons.py +408 -0
  63. memini/cli/push.py +877 -0
  64. memini/cli/recipes.py +103 -0
  65. memini/cli/replay.py +147 -0
  66. memini/cli/runs.py +199 -0
  67. memini/cli/search.py +155 -0
  68. memini/cli/serve.py +325 -0
  69. memini/cli/sources.py +995 -0
  70. memini/cli/status.py +224 -0
  71. memini/cli/tokens.py +187 -0
  72. memini/comms/__init__.py +88 -0
  73. memini/comms/model.py +408 -0
  74. memini/comms/service.py +888 -0
  75. memini/config.py +1265 -0
  76. memini/config_reference.py +915 -0
  77. memini/credentials/__init__.py +127 -0
  78. memini/credentials/crypto.py +205 -0
  79. memini/credentials/errors.py +134 -0
  80. memini/credentials/model.py +163 -0
  81. memini/credentials/rotation.py +139 -0
  82. memini/credentials/store.py +680 -0
  83. memini/db/__init__.py +40 -0
  84. memini/db/connection.py +89 -0
  85. memini/db/limits.py +116 -0
  86. memini/db/migrate.py +153 -0
  87. memini/db/migrations/0001_extensions.sql +7 -0
  88. memini/db/migrations/0002_core_schema.sql +287 -0
  89. memini/db/migrations/0003_jobs.sql +97 -0
  90. memini/db/migrations/0004_index_state.sql +57 -0
  91. memini/db/migrations/0005_chunks_and_symbols.sql +162 -0
  92. memini/db/migrations/0006_document_chunks.sql +15 -0
  93. memini/db/migrations/0007_credentials.sql +64 -0
  94. memini/db/migrations/0008_knowledge.sql +478 -0
  95. memini/db/migrations/0009_workspace_llm_policy.sql +19 -0
  96. memini/db/migrations/0010_lexical_stemming.sql +44 -0
  97. memini/db/migrations/0011_payload_chunks.sql +40 -0
  98. memini/db/migrations/0012_stream_history.sql +20 -0
  99. memini/db/migrations/0013_forget_and_retention.sql +77 -0
  100. memini/db/migrations/0014_persons.sql +125 -0
  101. memini/db/migrations/0015_raw_responses.sql +128 -0
  102. memini/db/migrations/0016_agent_runs.sql +139 -0
  103. memini/db/migrations/0017_knowledge_vectors.sql +37 -0
  104. memini/db/statements.py +174 -0
  105. memini/decisions/__init__.py +61 -0
  106. memini/decisions/model.py +332 -0
  107. memini/decisions/reads.py +74 -0
  108. memini/decisions/service.py +258 -0
  109. memini/embeddings/__init__.py +54 -0
  110. memini/embeddings/base.py +233 -0
  111. memini/embeddings/factory.py +88 -0
  112. memini/embeddings/fake.py +66 -0
  113. memini/embeddings/local.py +246 -0
  114. memini/embeddings/model_files.py +225 -0
  115. memini/embeddings/openai_compatible.py +242 -0
  116. memini/embeddings/reuse.py +99 -0
  117. memini/embeddings/runtime.py +186 -0
  118. memini/evaluation/__init__.py +79 -0
  119. memini/evaluation/decisions/__init__.py +79 -0
  120. memini/evaluation/decisions/candidates.py +156 -0
  121. memini/evaluation/decisions/dataset.py +695 -0
  122. memini/evaluation/decisions/llm.py +174 -0
  123. memini/evaluation/decisions/matching.py +236 -0
  124. memini/evaluation/decisions/report.py +560 -0
  125. memini/evaluation/decisions/runner.py +520 -0
  126. memini/evaluation/decisions/stubs.py +285 -0
  127. memini/evaluation/metrics.py +120 -0
  128. memini/evaluation/report.py +450 -0
  129. memini/evaluation/runner.py +586 -0
  130. memini/evaluation/suite.py +514 -0
  131. memini/evaluation/variants.py +300 -0
  132. memini/executors/__init__.py +27 -0
  133. memini/executors/agent/__init__.py +117 -0
  134. memini/executors/agent/model.py +360 -0
  135. memini/executors/agent/session.py +931 -0
  136. memini/executors/budget.py +107 -0
  137. memini/executors/client.py +388 -0
  138. memini/executors/core.py +245 -0
  139. memini/executors/egress.py +379 -0
  140. memini/executors/oauth.py +143 -0
  141. memini/executors/service.py +592 -0
  142. memini/executors/source.py +436 -0
  143. memini/executors/transport.py +277 -0
  144. memini/extraction/__init__.py +55 -0
  145. memini/extraction/candidates.py +315 -0
  146. memini/extraction/inputs.py +131 -0
  147. memini/extraction/job.py +504 -0
  148. memini/extraction/llm.py +683 -0
  149. memini/extraction/model.py +236 -0
  150. memini/extraction/reextract.py +439 -0
  151. memini/filters.py +432 -0
  152. memini/git/__init__.py +90 -0
  153. memini/git/command.py +500 -0
  154. memini/git/errors.py +110 -0
  155. memini/git/mirror.py +858 -0
  156. memini/git/options.py +85 -0
  157. memini/git/remote.py +201 -0
  158. memini/gitsync/__init__.py +79 -0
  159. memini/gitsync/content.py +206 -0
  160. memini/gitsync/handler.py +183 -0
  161. memini/gitsync/model.py +157 -0
  162. memini/gitsync/rules.py +157 -0
  163. memini/gitsync/sync.py +802 -0
  164. memini/ids.py +61 -0
  165. memini/index/__init__.py +111 -0
  166. memini/index/fingerprint.py +368 -0
  167. memini/index/raw.py +250 -0
  168. memini/index/rebuild.py +390 -0
  169. memini/index/vectors.py +223 -0
  170. memini/index/writer.py +536 -0
  171. memini/ingest/__init__.py +150 -0
  172. memini/ingest/checkpoint.py +63 -0
  173. memini/ingest/filtering.py +98 -0
  174. memini/ingest/indexing.py +47 -0
  175. memini/ingest/model.py +311 -0
  176. memini/ingest/store.py +520 -0
  177. memini/ingest/validate.py +389 -0
  178. memini/intake/__init__.py +91 -0
  179. memini/intake/plan.py +403 -0
  180. memini/intake/state.py +113 -0
  181. memini/jobs/__init__.py +62 -0
  182. memini/jobs/extraction.py +59 -0
  183. memini/jobs/queue.py +313 -0
  184. memini/jobs/scheduler.py +255 -0
  185. memini/jobs/worker.py +333 -0
  186. memini/knowledge/__init__.py +118 -0
  187. memini/knowledge/journal.py +451 -0
  188. memini/knowledge/model.py +439 -0
  189. memini/knowledge/reader.py +335 -0
  190. memini/llm/__init__.py +75 -0
  191. memini/llm/_endpoint.py +484 -0
  192. memini/llm/gate.py +371 -0
  193. memini/llm/model.py +170 -0
  194. memini/llm/workspace.py +74 -0
  195. memini/logs.py +420 -0
  196. memini/mcp/__init__.py +36 -0
  197. memini/mcp/comms.py +427 -0
  198. memini/mcp/decisions.py +284 -0
  199. memini/mcp/endpoint.py +69 -0
  200. memini/mcp/query.py +299 -0
  201. memini/mcp/server.py +178 -0
  202. memini/mcp/status.py +134 -0
  203. memini/mcp/text.py +62 -0
  204. memini/mcp/tools.py +122 -0
  205. memini/metrics.py +420 -0
  206. memini/payloads.py +768 -0
  207. memini/persons/__init__.py +146 -0
  208. memini/persons/directory.py +544 -0
  209. memini/persons/identities.py +302 -0
  210. memini/persons/lookup.py +174 -0
  211. memini/persons/model.py +197 -0
  212. memini/persons/resolve.py +370 -0
  213. memini/provenance.py +34 -0
  214. memini/prune.py +350 -0
  215. memini/py.typed +0 -0
  216. memini/query/__init__.py +81 -0
  217. memini/query/model.py +415 -0
  218. memini/query/reader.py +625 -0
  219. memini/query/service.py +369 -0
  220. memini/recipes/__init__.py +187 -0
  221. memini/recipes/bundled/__init__.py +9 -0
  222. memini/recipes/calls.py +195 -0
  223. memini/recipes/model.py +402 -0
  224. memini/recipes/parsing.py +324 -0
  225. memini/recipes/reading.py +908 -0
  226. memini/recipes/schema.py +23 -0
  227. memini/recipes/selectors.py +214 -0
  228. memini/recipes/templates.py +210 -0
  229. memini/recipes/times.py +331 -0
  230. memini/recipes/validate.py +655 -0
  231. memini/redaction/__init__.py +109 -0
  232. memini/redaction/detectors.py +398 -0
  233. memini/redaction/redact.py +176 -0
  234. memini/removal/__init__.py +74 -0
  235. memini/removal/core.py +102 -0
  236. memini/removal/forget.py +207 -0
  237. memini/removal/retention.py +183 -0
  238. memini/rerank/__init__.py +29 -0
  239. memini/rerank/factory.py +51 -0
  240. memini/rerank/local.py +358 -0
  241. memini/responses/__init__.py +139 -0
  242. memini/responses/replay.py +643 -0
  243. memini/responses/store.py +760 -0
  244. memini/runs.py +800 -0
  245. memini/search/__init__.py +71 -0
  246. memini/search/engine.py +371 -0
  247. memini/search/errors.py +32 -0
  248. memini/search/fusion.py +66 -0
  249. memini/search/model.py +419 -0
  250. memini/search/patterns.py +145 -0
  251. memini/search/retrieval.py +558 -0
  252. memini/sources.py +1340 -0
  253. memini/status.py +722 -0
  254. memini/token_admin.py +189 -0
  255. memini/workspaces.py +219 -0
  256. memini-0.1.0rc1.dist-info/METADATA +71 -0
  257. memini-0.1.0rc1.dist-info/RECORD +260 -0
  258. memini-0.1.0rc1.dist-info/WHEEL +4 -0
  259. memini-0.1.0rc1.dist-info/entry_points.txt +3 -0
  260. memini-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
memini/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Memini: a self-hosted knowledge service for code, messages and meeting notes."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ __version__: str = version("memini")
memini/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """``python -m memini``: the same as the ``memini`` command."""
2
+
3
+ from memini.cli import main
4
+
5
+ if __name__ == "__main__":
6
+ raise SystemExit(main())
memini/api/__init__.py ADDED
@@ -0,0 +1,19 @@
1
+ """The HTTP API: a FastAPI application built by :func:`create_app`."""
2
+
3
+ from memini.api.app import OPENAPI_URL, V1_PREFIX, create_app
4
+ from memini.api.errors import ApiError, ErrorResponse
5
+ from memini.api.health import DEFAULT_READINESS_TIMEOUT, ReadinessCheck
6
+ from memini.api.request_id import REQUEST_ID_HEADER, current_request_id, request_id_var
7
+
8
+ __all__ = [
9
+ "DEFAULT_READINESS_TIMEOUT",
10
+ "OPENAPI_URL",
11
+ "REQUEST_ID_HEADER",
12
+ "V1_PREFIX",
13
+ "ApiError",
14
+ "ErrorResponse",
15
+ "ReadinessCheck",
16
+ "create_app",
17
+ "current_request_id",
18
+ "request_id_var",
19
+ ]
@@ -0,0 +1,68 @@
1
+ """From the errors of the administration library to the API's error shape.
2
+
3
+ Wrap a call into :mod:`memini.workspaces`, :mod:`memini.sources`, :mod:`memini.token_admin` or
4
+ the run reads of :mod:`memini.runs` in :func:`admin_errors`. What each error becomes:
5
+
6
+ - :class:`~memini.workspaces.NotFoundError`: 404 ``not_found``. A workspace, source, stream, run or
7
+ token the token may not see is reported like one that does not exist.
8
+ - :class:`~memini.auth.WorkspaceLimitError`: 403 ``workspace_limited``: the token is limited to
9
+ one workspace and the write would reach beyond it.
10
+ - :class:`~memini.workspaces.WorkspaceExistsError`, :class:`~memini.sources.SourceExistsError`:
11
+ 409 ``conflict``.
12
+ - :class:`~memini.sources.SourceBusyError`: 409 ``source_busy``: a job of the source runs; try
13
+ again when it is done.
14
+ - :class:`~memini.sources.NotSyncableError`: 409 ``not_syncable``.
15
+ - :class:`~memini.sources.InvalidSourceError`: 422 ``invalid_source``. Its message names the
16
+ setting and the reason and never the value, so it is passed on: a location can hold a
17
+ credential, and it is not repeated.
18
+ """
19
+
20
+ from collections.abc import Generator
21
+ from contextlib import contextmanager
22
+
23
+ from memini.api.errors import ApiError
24
+ from memini.auth import WorkspaceLimitError
25
+ from memini.sources import (
26
+ InvalidSourceError,
27
+ NotSyncableError,
28
+ SourceBusyError,
29
+ SourceExistsError,
30
+ )
31
+ from memini.workspaces import NotFoundError, WorkspaceExistsError
32
+
33
+ __all__ = ["admin_errors"]
34
+
35
+
36
+ @contextmanager
37
+ def admin_errors() -> Generator[None]:
38
+ """Raise the :class:`ApiError` for an error of the administration library; see the module."""
39
+ try:
40
+ yield
41
+ except NotFoundError as error:
42
+ raise ApiError(404, "not_found", f"No such {error.what}.") from None
43
+ except WorkspaceLimitError:
44
+ raise ApiError(
45
+ 403,
46
+ "workspace_limited",
47
+ "The token is limited to one workspace; this needs a token that is not.",
48
+ ) from None
49
+ except WorkspaceExistsError:
50
+ raise ApiError(409, "conflict", "A workspace of that name exists.") from None
51
+ except SourceExistsError:
52
+ raise ApiError(409, "conflict", "The workspace has a source of that name.") from None
53
+ except SourceBusyError:
54
+ raise ApiError(
55
+ 409,
56
+ "source_busy",
57
+ "A job of the source is running; try again when it is done.",
58
+ ) from None
59
+ except NotSyncableError:
60
+ raise ApiError(
61
+ 409, "not_syncable", "Sources of this kind are not synced by the service."
62
+ ) from None
63
+ except InvalidSourceError as error:
64
+ raise ApiError(
65
+ 422,
66
+ "invalid_source",
67
+ f"The source setting {error.setting} is not valid: {error.reason}.",
68
+ ) from None
@@ -0,0 +1,469 @@
1
+ """The agent executor's routes: an intake of an mcp source driven from an interactive agent session
2
+ (:mod:`memini.executors.agent`). Under ``/v1/workspaces/{workspace}/sources/{source}/agent-runs``:
3
+
4
+ - ``POST`` starts a run of a stream: pins the database time, plans the windows, opens the first
5
+ one. 201 with the run. A stream with a run not finished: 409 ``agent_run_open``, whose message
6
+ names that run, to finish first.
7
+ - ``GET /{id}`` reads a run: its state, plan, windows and the counts of the session.
8
+ - ``GET /{id}/next`` is the next step: ``call``, a call to make exactly as given (the tool's name
9
+ at the server and its arguments) with the id to post its response under; ``window_done``, the
10
+ current window has nothing more to hand out (the next request opens the following one); or
11
+ ``run_done``, finish the run. A call that got no response is handed out again under the same id.
12
+ - ``POST /{id}/responses`` takes the raw result of a call handed out: its id, the tool and the
13
+ arguments as the agent sent them (they must be the call's: 409 ``call_mismatch`` otherwise),
14
+ and the ``result`` as the agent's client got it (an MCP result object, its list of content
15
+ blocks, or a text), or the ``error`` of a call that failed. A call the run did not hand out:
16
+ 409 ``call_not_handed_out``. Nothing is stored for a refused response. The answer is a receipt:
17
+ positions and counts, never a value of the response. Its body may be as large as
18
+ ``ingest_max_body_size``, as a batch's.
19
+ - ``POST /{id}/finish`` closes the run to calls, parses and ingests its windows, and writes the run
20
+ report (``GET /v1/runs/{run_id}``). A job of the source running: 409 ``source_busy``, send it
21
+ again later.
22
+
23
+ Every route needs the ``ingest`` scope and keeps a token limited to one workspace inside it: a
24
+ workspace, source, stream or run outside it is not found (404). A source that is not an mcp
25
+ source: 409 ``not_mcp_source``; one without a valid recipe or a registered parser: 409
26
+ ``source_not_ready``. Each ``POST`` counts against the token's write budget, ``next`` against its
27
+ reads (:mod:`memini.api.limits`).
28
+ """
29
+
30
+ from collections.abc import Generator
31
+ from contextlib import contextmanager
32
+ from datetime import datetime
33
+ from typing import Any, Literal, cast
34
+ from uuid import UUID
35
+
36
+ from fastapi import APIRouter, Request
37
+ from pydantic import BaseModel, ConfigDict, Field, JsonValue
38
+
39
+ from memini import ingest
40
+ from memini.api.auth import IngestAccess
41
+ from memini.api.database import Connection
42
+ from memini.api.errors import ApiError
43
+ from memini.api.ingest import BatchRoute, ingest_errors
44
+ from memini.config import Settings
45
+ from memini.db import Pool
46
+ from memini.executors import agent
47
+ from memini.extraction.job import extraction_enabled
48
+ from memini.query import QueryServices
49
+ from memini.recipes import ParserError
50
+ from memini.responses import (
51
+ InvalidResponseError,
52
+ NotAnMcpSourceError,
53
+ ReplayError,
54
+ ResponseTooLargeError,
55
+ ServerInfo,
56
+ WindowClosedError,
57
+ WindowNotFoundError,
58
+ )
59
+ from memini.sources import SourceBusyError
60
+ from memini.workspaces import NotFoundError
61
+
62
+ __all__ = [
63
+ "AgentRunInfo",
64
+ "CallInfo",
65
+ "FinishAnswer",
66
+ "NextAnswer",
67
+ "ReceiptAnswer",
68
+ "ResponseBody",
69
+ "StartBody",
70
+ "WindowInfo",
71
+ "build_agent_runs_router",
72
+ ]
73
+
74
+ _MAX_TEXT = 1_000_000
75
+
76
+
77
+ # --- Bodies --------------------------------------------------------------------------------------
78
+
79
+
80
+ class ServerBody(BaseModel):
81
+ """What the session's server reported when it started (``serverInfo``)."""
82
+
83
+ model_config = ConfigDict(extra="forbid")
84
+
85
+ name: str = Field(min_length=1, max_length=256, description="serverInfo.name.")
86
+ version: str | None = Field(
87
+ default=None, min_length=1, max_length=128, description="serverInfo.version."
88
+ )
89
+
90
+
91
+ class StartBody(BaseModel):
92
+ model_config = ConfigDict(extra="forbid")
93
+
94
+ stream: str = Field(
95
+ min_length=1, max_length=1024, description="The key of one of the source's streams."
96
+ )
97
+ server: ServerBody | None = Field(
98
+ default=None,
99
+ description="The session's server, if the agent can tell: another name than the recipe's"
100
+ " is refused. Left out: the recipe's server, version unknown.",
101
+ )
102
+
103
+
104
+ class ResponseBody(BaseModel):
105
+ """The raw result of a call handed out, as the agent's client got it."""
106
+
107
+ model_config = ConfigDict(extra="forbid")
108
+
109
+ call_id: UUID = Field(description="The id the call was handed out with.")
110
+ tool: str = Field(
111
+ min_length=1, max_length=128, description="The tool's name at the server, as handed out."
112
+ )
113
+ arguments: dict[str, Any] = Field(description="The arguments the call was made with.")
114
+ result: JsonValue = Field(
115
+ default=None,
116
+ description="The tool's result: an MCP result object (content, structuredContent,"
117
+ " isError), the list of its content blocks, or its text. Stored redacted, otherwise"
118
+ " verbatim.",
119
+ )
120
+ error: str | None = Field(
121
+ default=None,
122
+ max_length=_MAX_TEXT,
123
+ description="Instead of result: the error of a call that failed. The call is then handed"
124
+ " out again, a few times.",
125
+ )
126
+
127
+
128
+ # --- Answers -------------------------------------------------------------------------------------
129
+
130
+
131
+ class WindowInfo(BaseModel):
132
+ position: int = Field(description="The window's place in the plan, from 0.")
133
+ kind: Literal["initial", "forward", "backfill"]
134
+ start: datetime = Field(serialization_alias="from")
135
+ end: datetime = Field(serialization_alias="to")
136
+ status: agent.WindowStatus
137
+ complete: bool | None = Field(description="Its claim once finish ingested it.")
138
+
139
+ @classmethod
140
+ def of(cls, window: agent.AgentWindow) -> "WindowInfo":
141
+ return cls(
142
+ position=window.position,
143
+ kind=window.kind,
144
+ start=window.start,
145
+ end=window.end,
146
+ status=window.status,
147
+ complete=window.complete,
148
+ )
149
+
150
+
151
+ class AgentRunInfo(BaseModel):
152
+ id: UUID
153
+ workspace: str
154
+ source: str
155
+ stream: str
156
+ state: Literal["open", "finishing", "finished"]
157
+ as_of: datetime = Field(description="The database time pinned for the run.")
158
+ server: str
159
+ server_version: str | None
160
+ recipe_digest: str
161
+ started_at: datetime
162
+ closed_at: datetime | None
163
+ finished_at: datetime | None
164
+ run_id: UUID | None = Field(description="The run report, once finished: GET /v1/runs/{id}.")
165
+ plan: dict[str, JsonValue]
166
+ windows: list[WindowInfo]
167
+ counts: dict[str, int] = Field(description="Calls handed out, responses stored, and so on.")
168
+
169
+ @classmethod
170
+ def of(cls, run: agent.AgentRun) -> "AgentRunInfo":
171
+ return cls(
172
+ id=run.id,
173
+ workspace=run.workspace,
174
+ source=run.source,
175
+ stream=run.stream,
176
+ state=run.state,
177
+ as_of=run.as_of,
178
+ server=run.server.name,
179
+ server_version=run.server.version,
180
+ recipe_digest=run.recipe_digest,
181
+ started_at=run.started_at,
182
+ closed_at=run.closed_at,
183
+ finished_at=run.finished_at,
184
+ run_id=run.run_id,
185
+ plan=dict(run.plan),
186
+ windows=[WindowInfo.of(window) for window in run.windows],
187
+ counts=dict(run.counts),
188
+ )
189
+
190
+
191
+ class CallInfo(BaseModel):
192
+ id: UUID = Field(description="Post the call's response under this id.")
193
+ call: Literal["list", "detail"]
194
+ tool: str = Field(description="The tool's name at the server.")
195
+ arguments: dict[str, Any] = Field(description="Send exactly these, unchanged.")
196
+ page: int
197
+ item_id: str | None
198
+ check: str = Field(description="The completeness check the call makes pass.")
199
+ reason: str = Field(description="Why the call is needed, by position.")
200
+ attempt: int = Field(description="1 the first time this call of the window is handed out.")
201
+
202
+ @classmethod
203
+ def of(cls, call: agent.HandedOutCall) -> "CallInfo":
204
+ return cls(
205
+ id=call.id,
206
+ call=call.call,
207
+ tool=call.tool,
208
+ arguments=dict(call.arguments),
209
+ page=call.page,
210
+ item_id=call.item_id,
211
+ check=call.check,
212
+ reason=call.reason,
213
+ attempt=call.attempt,
214
+ )
215
+
216
+
217
+ class NextAnswer(BaseModel):
218
+ status: agent.NextStatus = Field(
219
+ description="call: make the call. window_done: ask again for the next window."
220
+ " run_done: finish the run."
221
+ )
222
+ call: CallInfo | None = None
223
+ window: WindowInfo | None = None
224
+ complete: bool | None = Field(
225
+ default=None,
226
+ description="window_done: whether the window passes every check, as far as the stored"
227
+ " responses tell before the parser runs.",
228
+ )
229
+ left: int = Field(
230
+ default=0, description="window_done: calls the window misses that are not handed out."
231
+ )
232
+ reason: str | None = Field(default=None, description="run_done: why.")
233
+
234
+ @classmethod
235
+ def of(cls, step: agent.NextStep) -> "NextAnswer":
236
+ return cls(
237
+ status=step.status,
238
+ call=None if step.call is None else CallInfo.of(step.call),
239
+ window=None if step.window is None else WindowInfo.of(step.window),
240
+ complete=step.complete,
241
+ left=step.left,
242
+ reason=step.reason,
243
+ )
244
+
245
+
246
+ class ReceiptAnswer(BaseModel):
247
+ """What storing the response did; never a value of it."""
248
+
249
+ call_id: UUID
250
+ window: WindowInfo
251
+ position: int = Field(description="The response's place in its window, from 1.")
252
+ stored: bool = Field(description="False: the window held the same result for the call.")
253
+ is_error: bool = Field(description="The call failed; it is handed out again, a few times.")
254
+ responses: int = Field(description="Responses the window holds now.")
255
+ redacted: int = Field(description="Values redacted from the result and the arguments.")
256
+
257
+ @classmethod
258
+ def of(cls, receipt: agent.Receipt) -> "ReceiptAnswer":
259
+ return cls(
260
+ call_id=receipt.call_id,
261
+ window=WindowInfo.of(receipt.window),
262
+ position=receipt.position,
263
+ stored=receipt.stored,
264
+ is_error=receipt.is_error,
265
+ responses=receipt.responses,
266
+ redacted=receipt.redacted,
267
+ )
268
+
269
+
270
+ class WindowResult(BaseModel):
271
+ window: WindowInfo
272
+ ingested: bool = Field(description="False: never opened, skipped or gone.")
273
+ records: int = 0
274
+ inserted: int = 0
275
+ updated: int = 0
276
+ revived: int = 0
277
+ deleted: int = 0
278
+ unchanged: int = 0
279
+ reconciled: int = 0
280
+ checkpoint_moved: bool = False
281
+ incomplete: list[str] = Field(default_factory=list[str], description="Why it is incomplete.")
282
+ missing_calls: int = Field(default=0, description="Calls that would complete it.")
283
+
284
+ @classmethod
285
+ def of(cls, outcome: agent.WindowOutcome) -> "WindowResult":
286
+ done = outcome.ingest
287
+ info = WindowInfo.of(outcome.window)
288
+ if done is None:
289
+ return cls(window=info, ingested=False)
290
+ return cls(
291
+ window=info,
292
+ ingested=True,
293
+ records=done.records,
294
+ inserted=done.inserted,
295
+ updated=done.updated,
296
+ revived=done.revived,
297
+ deleted=done.deleted,
298
+ unchanged=done.unchanged,
299
+ reconciled=done.reconciled,
300
+ checkpoint_moved=done.checkpoint_moved,
301
+ incomplete=list(done.incomplete),
302
+ missing_calls=len(done.missing),
303
+ )
304
+
305
+
306
+ class FinishAnswer(BaseModel):
307
+ agent_run: AgentRunInfo
308
+ run_id: UUID = Field(description="The run report: GET /v1/runs/{id}.")
309
+ windows: list[WindowResult]
310
+ checked_until: datetime | None = Field(description="The stream's checkpoint now.")
311
+
312
+ @classmethod
313
+ def of(cls, result: agent.FinishResult) -> "FinishAnswer":
314
+ return cls(
315
+ agent_run=AgentRunInfo.of(result.agent_run),
316
+ run_id=result.run_id,
317
+ windows=[WindowResult.of(outcome) for outcome in result.windows],
318
+ checked_until=result.checked_until,
319
+ )
320
+
321
+
322
+ # --- Errors --------------------------------------------------------------------------------------
323
+
324
+
325
+ @contextmanager
326
+ def _agent_errors() -> Generator[None]:
327
+ try:
328
+ yield
329
+ except NotFoundError as error:
330
+ raise ApiError(404, "not_found", f"No such {error.what}.") from None
331
+ except NotAnMcpSourceError:
332
+ raise ApiError(
333
+ 409, "not_mcp_source", "Agent runs are for mcp sources; this source is not one."
334
+ ) from None
335
+ except agent.SourceNotReadyError as error:
336
+ raise ApiError(409, "source_not_ready", f"The source cannot run: {error}.") from None
337
+ except agent.ServerMismatchError as error:
338
+ raise ApiError(409, "server_mismatch", f"{_sentence(error)}.") from None
339
+ except agent.AgentRunOpenError as error:
340
+ raise ApiError(409, "agent_run_open", f"{_sentence(error)}.") from None
341
+ except agent.AgentRunClosedError as error:
342
+ code = "agent_run_finished" if error.finished else "agent_run_finishing"
343
+ raise ApiError(409, code, f"{_sentence(error)}.") from None
344
+ except agent.RecipeChangedError as error:
345
+ raise ApiError(409, "recipe_changed", f"{_sentence(error)}.") from None
346
+ except agent.CallNotHandedOutError as error:
347
+ raise ApiError(409, "call_not_handed_out", f"{_sentence(error)}; nothing stored.") from None
348
+ except agent.CallMismatchError as error:
349
+ raise ApiError(409, "call_mismatch", f"{_sentence(error)}; nothing stored.") from None
350
+ except agent.InvalidResultError as error:
351
+ raise ApiError(422, "invalid_result", f"{_sentence(error)}.") from None
352
+ except ResponseTooLargeError as error:
353
+ raise ApiError(413, "response_too_large", f"The response is too large: {error}.") from None
354
+ except InvalidResponseError as error:
355
+ raise ApiError(
356
+ 422, "invalid_response", f"The response cannot be stored: {error}."
357
+ ) from None
358
+ except WindowNotFoundError:
359
+ raise ApiError(
360
+ 409, "window_gone", "The call's window was deleted; finish the run and start another."
361
+ ) from None
362
+ except WindowClosedError:
363
+ raise ApiError(409, "window_closed", "The call's window was parsed already.") from None
364
+ except SourceBusyError as error:
365
+ raise ApiError(409, "source_busy", f"{_sentence(error)}.") from None
366
+ except ReplayError as error:
367
+ raise ApiError(409, "recipe_mismatch", f"{_sentence(error)}.") from None
368
+ except ParserError as error:
369
+ raise ApiError(409, "parser_failed", f"The parser failed: {error}.") from None
370
+
371
+
372
+ def _sentence(error: Exception) -> str:
373
+ text = str(error)
374
+ return text[:1].upper() + text[1:]
375
+
376
+
377
+ # --- Routes --------------------------------------------------------------------------------------
378
+
379
+
380
+ def build_agent_runs_router() -> APIRouter:
381
+ """The agent executor's routes; see the module."""
382
+ router = APIRouter(
383
+ prefix="/workspaces/{workspace}/sources/{source}/agent-runs",
384
+ tags=["agent runs"],
385
+ route_class=BatchRoute,
386
+ )
387
+
388
+ @router.post("", status_code=201, summary="Start an agent run of a stream")
389
+ async def start_agent_run(
390
+ workspace: str, source: str, body: StartBody, access: IngestAccess, conn: Connection
391
+ ) -> AgentRunInfo:
392
+ """Pin the database time, plan the stream's windows and open the first one."""
393
+ server = None if body.server is None else ServerInfo(body.server.name, body.server.version)
394
+ with _agent_errors():
395
+ run = await agent.start_run(conn, access, workspace, source, body.stream, server=server)
396
+ return AgentRunInfo.of(run)
397
+
398
+ @router.get("/{run_id}", summary="Read an agent run")
399
+ async def read_agent_run(
400
+ workspace: str, source: str, run_id: UUID, access: IngestAccess, conn: Connection
401
+ ) -> AgentRunInfo:
402
+ with _agent_errors():
403
+ run = await agent.get_agent_run(conn, access, workspace, source, run_id)
404
+ return AgentRunInfo.of(run)
405
+
406
+ @router.get("/{run_id}/next", summary="The next call of an agent run")
407
+ async def next_agent_call(
408
+ workspace: str, source: str, run_id: UUID, access: IngestAccess, conn: Connection
409
+ ) -> NextAnswer:
410
+ """The next call to make, or that the window or the run is done."""
411
+ with _agent_errors():
412
+ step = await agent.next_call(conn, access, workspace, source, run_id)
413
+ return NextAnswer.of(step)
414
+
415
+ @router.post("/{run_id}/responses", summary="Store the raw response of a call handed out")
416
+ async def store_agent_response(
417
+ workspace: str,
418
+ source: str,
419
+ run_id: UUID,
420
+ body: ResponseBody,
421
+ access: IngestAccess,
422
+ conn: Connection,
423
+ ) -> ReceiptAnswer:
424
+ """Store the result of a call this run handed out, redacted and otherwise verbatim. The
425
+ answer counts; it never repeats the response."""
426
+ with _agent_errors():
427
+ receipt = await agent.submit_response(
428
+ conn,
429
+ access,
430
+ workspace,
431
+ source,
432
+ run_id,
433
+ body.call_id,
434
+ tool=body.tool,
435
+ arguments=body.arguments,
436
+ result=body.result,
437
+ error=body.error,
438
+ )
439
+ return ReceiptAnswer.of(receipt)
440
+
441
+ @router.post("/{run_id}/finish", summary="Finish an agent run")
442
+ async def finish_agent_run(
443
+ workspace: str,
444
+ source: str,
445
+ run_id: UUID,
446
+ access: IngestAccess,
447
+ request: Request,
448
+ conn: Connection,
449
+ ) -> FinishAnswer:
450
+ """Close the run to calls, parse and ingest its windows, write the run report."""
451
+ settings = cast(Settings, request.app.state.settings)
452
+ pool = cast(Pool, request.app.state.pool)
453
+ embedder = cast(QueryServices, request.app.state.services).embedder
454
+ indexer = None if embedder is None else ingest.PayloadIndexer(embedder)
455
+ with _agent_errors(), ingest_errors():
456
+ result = await agent.finish_run(
457
+ conn,
458
+ access,
459
+ workspace,
460
+ source,
461
+ run_id,
462
+ runs=pool,
463
+ limits=ingest.IngestLimits.from_settings(settings),
464
+ indexer=indexer,
465
+ extract=extraction_enabled(settings),
466
+ )
467
+ return FinishAnswer.of(result)
468
+
469
+ return router