memini 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memini/__init__.py +5 -0
- memini/__main__.py +6 -0
- memini/api/__init__.py +19 -0
- memini/api/admin_errors.py +68 -0
- memini/api/agent_runs.py +469 -0
- memini/api/app.py +276 -0
- memini/api/auth.py +192 -0
- memini/api/comms.py +191 -0
- memini/api/database.py +30 -0
- memini/api/decisions.py +140 -0
- memini/api/errors.py +224 -0
- memini/api/extract.py +178 -0
- memini/api/forget.py +156 -0
- memini/api/health.py +129 -0
- memini/api/ingest.py +469 -0
- memini/api/limits.py +503 -0
- memini/api/mcp.py +169 -0
- memini/api/metrics.py +422 -0
- memini/api/persons.py +373 -0
- memini/api/query.py +148 -0
- memini/api/request_id.py +79 -0
- memini/api/runs.py +119 -0
- memini/api/sources.py +978 -0
- memini/api/status.py +71 -0
- memini/api/tokens.py +158 -0
- memini/api/workspaces.py +79 -0
- memini/auth/__init__.py +72 -0
- memini/auth/authentication.py +156 -0
- memini/auth/authorization.py +205 -0
- memini/auth/principal.py +45 -0
- memini/auth/tokens.py +243 -0
- memini/bench/__init__.py +59 -0
- memini/bench/corpus.py +277 -0
- memini/bench/measure.py +124 -0
- memini/bench/queries.py +134 -0
- memini/bench/report.py +310 -0
- memini/bench/repository.py +213 -0
- memini/bench/runner.py +1086 -0
- memini/chunking/__init__.py +194 -0
- memini/chunking/chunker.py +584 -0
- memini/chunking/languages.py +414 -0
- memini/chunking/markdown.py +393 -0
- memini/chunking/model.py +213 -0
- memini/chunking/payloads.py +453 -0
- memini/chunking/text.py +163 -0
- memini/cli/__init__.py +597 -0
- memini/cli/admin.py +191 -0
- memini/cli/arguments.py +248 -0
- memini/cli/bench.py +321 -0
- memini/cli/client.py +493 -0
- memini/cli/common.py +221 -0
- memini/cli/comms.py +181 -0
- memini/cli/credentials.py +385 -0
- memini/cli/decisions.py +341 -0
- memini/cli/evaluate.py +193 -0
- memini/cli/evaluate_decisions.py +225 -0
- memini/cli/export.py +116 -0
- memini/cli/extract.py +102 -0
- memini/cli/forget.py +96 -0
- memini/cli/index.py +89 -0
- memini/cli/output.py +117 -0
- memini/cli/persons.py +408 -0
- memini/cli/push.py +877 -0
- memini/cli/recipes.py +103 -0
- memini/cli/replay.py +147 -0
- memini/cli/runs.py +199 -0
- memini/cli/search.py +155 -0
- memini/cli/serve.py +325 -0
- memini/cli/sources.py +995 -0
- memini/cli/status.py +224 -0
- memini/cli/tokens.py +187 -0
- memini/comms/__init__.py +88 -0
- memini/comms/model.py +408 -0
- memini/comms/service.py +888 -0
- memini/config.py +1265 -0
- memini/config_reference.py +915 -0
- memini/credentials/__init__.py +127 -0
- memini/credentials/crypto.py +205 -0
- memini/credentials/errors.py +134 -0
- memini/credentials/model.py +163 -0
- memini/credentials/rotation.py +139 -0
- memini/credentials/store.py +680 -0
- memini/db/__init__.py +40 -0
- memini/db/connection.py +89 -0
- memini/db/limits.py +116 -0
- memini/db/migrate.py +153 -0
- memini/db/migrations/0001_extensions.sql +7 -0
- memini/db/migrations/0002_core_schema.sql +287 -0
- memini/db/migrations/0003_jobs.sql +97 -0
- memini/db/migrations/0004_index_state.sql +57 -0
- memini/db/migrations/0005_chunks_and_symbols.sql +162 -0
- memini/db/migrations/0006_document_chunks.sql +15 -0
- memini/db/migrations/0007_credentials.sql +64 -0
- memini/db/migrations/0008_knowledge.sql +478 -0
- memini/db/migrations/0009_workspace_llm_policy.sql +19 -0
- memini/db/migrations/0010_lexical_stemming.sql +44 -0
- memini/db/migrations/0011_payload_chunks.sql +40 -0
- memini/db/migrations/0012_stream_history.sql +20 -0
- memini/db/migrations/0013_forget_and_retention.sql +77 -0
- memini/db/migrations/0014_persons.sql +125 -0
- memini/db/migrations/0015_raw_responses.sql +128 -0
- memini/db/migrations/0016_agent_runs.sql +139 -0
- memini/db/migrations/0017_knowledge_vectors.sql +37 -0
- memini/db/statements.py +174 -0
- memini/decisions/__init__.py +61 -0
- memini/decisions/model.py +332 -0
- memini/decisions/reads.py +74 -0
- memini/decisions/service.py +258 -0
- memini/embeddings/__init__.py +54 -0
- memini/embeddings/base.py +233 -0
- memini/embeddings/factory.py +88 -0
- memini/embeddings/fake.py +66 -0
- memini/embeddings/local.py +246 -0
- memini/embeddings/model_files.py +225 -0
- memini/embeddings/openai_compatible.py +242 -0
- memini/embeddings/reuse.py +99 -0
- memini/embeddings/runtime.py +186 -0
- memini/evaluation/__init__.py +79 -0
- memini/evaluation/decisions/__init__.py +79 -0
- memini/evaluation/decisions/candidates.py +156 -0
- memini/evaluation/decisions/dataset.py +695 -0
- memini/evaluation/decisions/llm.py +174 -0
- memini/evaluation/decisions/matching.py +236 -0
- memini/evaluation/decisions/report.py +560 -0
- memini/evaluation/decisions/runner.py +520 -0
- memini/evaluation/decisions/stubs.py +285 -0
- memini/evaluation/metrics.py +120 -0
- memini/evaluation/report.py +450 -0
- memini/evaluation/runner.py +586 -0
- memini/evaluation/suite.py +514 -0
- memini/evaluation/variants.py +300 -0
- memini/executors/__init__.py +27 -0
- memini/executors/agent/__init__.py +117 -0
- memini/executors/agent/model.py +360 -0
- memini/executors/agent/session.py +931 -0
- memini/executors/budget.py +107 -0
- memini/executors/client.py +388 -0
- memini/executors/core.py +245 -0
- memini/executors/egress.py +379 -0
- memini/executors/oauth.py +143 -0
- memini/executors/service.py +592 -0
- memini/executors/source.py +436 -0
- memini/executors/transport.py +277 -0
- memini/extraction/__init__.py +55 -0
- memini/extraction/candidates.py +315 -0
- memini/extraction/inputs.py +131 -0
- memini/extraction/job.py +504 -0
- memini/extraction/llm.py +683 -0
- memini/extraction/model.py +236 -0
- memini/extraction/reextract.py +439 -0
- memini/filters.py +432 -0
- memini/git/__init__.py +90 -0
- memini/git/command.py +500 -0
- memini/git/errors.py +110 -0
- memini/git/mirror.py +858 -0
- memini/git/options.py +85 -0
- memini/git/remote.py +201 -0
- memini/gitsync/__init__.py +79 -0
- memini/gitsync/content.py +206 -0
- memini/gitsync/handler.py +183 -0
- memini/gitsync/model.py +157 -0
- memini/gitsync/rules.py +157 -0
- memini/gitsync/sync.py +802 -0
- memini/ids.py +61 -0
- memini/index/__init__.py +111 -0
- memini/index/fingerprint.py +368 -0
- memini/index/raw.py +250 -0
- memini/index/rebuild.py +390 -0
- memini/index/vectors.py +223 -0
- memini/index/writer.py +536 -0
- memini/ingest/__init__.py +150 -0
- memini/ingest/checkpoint.py +63 -0
- memini/ingest/filtering.py +98 -0
- memini/ingest/indexing.py +47 -0
- memini/ingest/model.py +311 -0
- memini/ingest/store.py +520 -0
- memini/ingest/validate.py +389 -0
- memini/intake/__init__.py +91 -0
- memini/intake/plan.py +403 -0
- memini/intake/state.py +113 -0
- memini/jobs/__init__.py +62 -0
- memini/jobs/extraction.py +59 -0
- memini/jobs/queue.py +313 -0
- memini/jobs/scheduler.py +255 -0
- memini/jobs/worker.py +333 -0
- memini/knowledge/__init__.py +118 -0
- memini/knowledge/journal.py +451 -0
- memini/knowledge/model.py +439 -0
- memini/knowledge/reader.py +335 -0
- memini/llm/__init__.py +75 -0
- memini/llm/_endpoint.py +484 -0
- memini/llm/gate.py +371 -0
- memini/llm/model.py +170 -0
- memini/llm/workspace.py +74 -0
- memini/logs.py +420 -0
- memini/mcp/__init__.py +36 -0
- memini/mcp/comms.py +427 -0
- memini/mcp/decisions.py +284 -0
- memini/mcp/endpoint.py +69 -0
- memini/mcp/query.py +299 -0
- memini/mcp/server.py +178 -0
- memini/mcp/status.py +134 -0
- memini/mcp/text.py +62 -0
- memini/mcp/tools.py +122 -0
- memini/metrics.py +420 -0
- memini/payloads.py +768 -0
- memini/persons/__init__.py +146 -0
- memini/persons/directory.py +544 -0
- memini/persons/identities.py +302 -0
- memini/persons/lookup.py +174 -0
- memini/persons/model.py +197 -0
- memini/persons/resolve.py +370 -0
- memini/provenance.py +34 -0
- memini/prune.py +350 -0
- memini/py.typed +0 -0
- memini/query/__init__.py +81 -0
- memini/query/model.py +415 -0
- memini/query/reader.py +625 -0
- memini/query/service.py +369 -0
- memini/recipes/__init__.py +187 -0
- memini/recipes/bundled/__init__.py +9 -0
- memini/recipes/calls.py +195 -0
- memini/recipes/model.py +402 -0
- memini/recipes/parsing.py +324 -0
- memini/recipes/reading.py +908 -0
- memini/recipes/schema.py +23 -0
- memini/recipes/selectors.py +214 -0
- memini/recipes/templates.py +210 -0
- memini/recipes/times.py +331 -0
- memini/recipes/validate.py +655 -0
- memini/redaction/__init__.py +109 -0
- memini/redaction/detectors.py +398 -0
- memini/redaction/redact.py +176 -0
- memini/removal/__init__.py +74 -0
- memini/removal/core.py +102 -0
- memini/removal/forget.py +207 -0
- memini/removal/retention.py +183 -0
- memini/rerank/__init__.py +29 -0
- memini/rerank/factory.py +51 -0
- memini/rerank/local.py +358 -0
- memini/responses/__init__.py +139 -0
- memini/responses/replay.py +643 -0
- memini/responses/store.py +760 -0
- memini/runs.py +800 -0
- memini/search/__init__.py +71 -0
- memini/search/engine.py +371 -0
- memini/search/errors.py +32 -0
- memini/search/fusion.py +66 -0
- memini/search/model.py +419 -0
- memini/search/patterns.py +145 -0
- memini/search/retrieval.py +558 -0
- memini/sources.py +1340 -0
- memini/status.py +722 -0
- memini/token_admin.py +189 -0
- memini/workspaces.py +219 -0
- memini-0.1.0rc1.dist-info/METADATA +71 -0
- memini-0.1.0rc1.dist-info/RECORD +260 -0
- memini-0.1.0rc1.dist-info/WHEEL +4 -0
- memini-0.1.0rc1.dist-info/entry_points.txt +3 -0
- memini-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
memini/__init__.py
ADDED
memini/__main__.py
ADDED
memini/api/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""The HTTP API: a FastAPI application built by :func:`create_app`."""
|
|
2
|
+
|
|
3
|
+
from memini.api.app import OPENAPI_URL, V1_PREFIX, create_app
|
|
4
|
+
from memini.api.errors import ApiError, ErrorResponse
|
|
5
|
+
from memini.api.health import DEFAULT_READINESS_TIMEOUT, ReadinessCheck
|
|
6
|
+
from memini.api.request_id import REQUEST_ID_HEADER, current_request_id, request_id_var
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"DEFAULT_READINESS_TIMEOUT",
|
|
10
|
+
"OPENAPI_URL",
|
|
11
|
+
"REQUEST_ID_HEADER",
|
|
12
|
+
"V1_PREFIX",
|
|
13
|
+
"ApiError",
|
|
14
|
+
"ErrorResponse",
|
|
15
|
+
"ReadinessCheck",
|
|
16
|
+
"create_app",
|
|
17
|
+
"current_request_id",
|
|
18
|
+
"request_id_var",
|
|
19
|
+
]
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""From the errors of the administration library to the API's error shape.
|
|
2
|
+
|
|
3
|
+
Wrap a call into :mod:`memini.workspaces`, :mod:`memini.sources`, :mod:`memini.token_admin` or
|
|
4
|
+
the run reads of :mod:`memini.runs` in :func:`admin_errors`. What each error becomes:
|
|
5
|
+
|
|
6
|
+
- :class:`~memini.workspaces.NotFoundError`: 404 ``not_found``. A workspace, source, stream, run or
|
|
7
|
+
token the token may not see is reported like one that does not exist.
|
|
8
|
+
- :class:`~memini.auth.WorkspaceLimitError`: 403 ``workspace_limited``: the token is limited to
|
|
9
|
+
one workspace and the write would reach beyond it.
|
|
10
|
+
- :class:`~memini.workspaces.WorkspaceExistsError`, :class:`~memini.sources.SourceExistsError`:
|
|
11
|
+
409 ``conflict``.
|
|
12
|
+
- :class:`~memini.sources.SourceBusyError`: 409 ``source_busy``: a job of the source runs; try
|
|
13
|
+
again when it is done.
|
|
14
|
+
- :class:`~memini.sources.NotSyncableError`: 409 ``not_syncable``.
|
|
15
|
+
- :class:`~memini.sources.InvalidSourceError`: 422 ``invalid_source``. Its message names the
|
|
16
|
+
setting and the reason and never the value, so it is passed on: a location can hold a
|
|
17
|
+
credential, and it is not repeated.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from collections.abc import Generator
|
|
21
|
+
from contextlib import contextmanager
|
|
22
|
+
|
|
23
|
+
from memini.api.errors import ApiError
|
|
24
|
+
from memini.auth import WorkspaceLimitError
|
|
25
|
+
from memini.sources import (
|
|
26
|
+
InvalidSourceError,
|
|
27
|
+
NotSyncableError,
|
|
28
|
+
SourceBusyError,
|
|
29
|
+
SourceExistsError,
|
|
30
|
+
)
|
|
31
|
+
from memini.workspaces import NotFoundError, WorkspaceExistsError
|
|
32
|
+
|
|
33
|
+
__all__ = ["admin_errors"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@contextmanager
|
|
37
|
+
def admin_errors() -> Generator[None]:
|
|
38
|
+
"""Raise the :class:`ApiError` for an error of the administration library; see the module."""
|
|
39
|
+
try:
|
|
40
|
+
yield
|
|
41
|
+
except NotFoundError as error:
|
|
42
|
+
raise ApiError(404, "not_found", f"No such {error.what}.") from None
|
|
43
|
+
except WorkspaceLimitError:
|
|
44
|
+
raise ApiError(
|
|
45
|
+
403,
|
|
46
|
+
"workspace_limited",
|
|
47
|
+
"The token is limited to one workspace; this needs a token that is not.",
|
|
48
|
+
) from None
|
|
49
|
+
except WorkspaceExistsError:
|
|
50
|
+
raise ApiError(409, "conflict", "A workspace of that name exists.") from None
|
|
51
|
+
except SourceExistsError:
|
|
52
|
+
raise ApiError(409, "conflict", "The workspace has a source of that name.") from None
|
|
53
|
+
except SourceBusyError:
|
|
54
|
+
raise ApiError(
|
|
55
|
+
409,
|
|
56
|
+
"source_busy",
|
|
57
|
+
"A job of the source is running; try again when it is done.",
|
|
58
|
+
) from None
|
|
59
|
+
except NotSyncableError:
|
|
60
|
+
raise ApiError(
|
|
61
|
+
409, "not_syncable", "Sources of this kind are not synced by the service."
|
|
62
|
+
) from None
|
|
63
|
+
except InvalidSourceError as error:
|
|
64
|
+
raise ApiError(
|
|
65
|
+
422,
|
|
66
|
+
"invalid_source",
|
|
67
|
+
f"The source setting {error.setting} is not valid: {error.reason}.",
|
|
68
|
+
) from None
|
memini/api/agent_runs.py
ADDED
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
"""The agent executor's routes: an intake of an mcp source driven from an interactive agent session
|
|
2
|
+
(:mod:`memini.executors.agent`). Under ``/v1/workspaces/{workspace}/sources/{source}/agent-runs``:
|
|
3
|
+
|
|
4
|
+
- ``POST`` starts a run of a stream: pins the database time, plans the windows, opens the first
|
|
5
|
+
one. 201 with the run. A stream with a run not finished: 409 ``agent_run_open``, whose message
|
|
6
|
+
names that run, to finish first.
|
|
7
|
+
- ``GET /{id}`` reads a run: its state, plan, windows and the counts of the session.
|
|
8
|
+
- ``GET /{id}/next`` is the next step: ``call``, a call to make exactly as given (the tool's name
|
|
9
|
+
at the server and its arguments) with the id to post its response under; ``window_done``, the
|
|
10
|
+
current window has nothing more to hand out (the next request opens the following one); or
|
|
11
|
+
``run_done``, finish the run. A call that got no response is handed out again under the same id.
|
|
12
|
+
- ``POST /{id}/responses`` takes the raw result of a call handed out: its id, the tool and the
|
|
13
|
+
arguments as the agent sent them (they must be the call's: 409 ``call_mismatch`` otherwise),
|
|
14
|
+
and the ``result`` as the agent's client got it (an MCP result object, its list of content
|
|
15
|
+
blocks, or a text), or the ``error`` of a call that failed. A call the run did not hand out:
|
|
16
|
+
409 ``call_not_handed_out``. Nothing is stored for a refused response. The answer is a receipt:
|
|
17
|
+
positions and counts, never a value of the response. Its body may be as large as
|
|
18
|
+
``ingest_max_body_size``, as a batch's.
|
|
19
|
+
- ``POST /{id}/finish`` closes the run to calls, parses and ingests its windows, and writes the run
|
|
20
|
+
report (``GET /v1/runs/{run_id}``). A job of the source running: 409 ``source_busy``, send it
|
|
21
|
+
again later.
|
|
22
|
+
|
|
23
|
+
Every route needs the ``ingest`` scope and keeps a token limited to one workspace inside it: a
|
|
24
|
+
workspace, source, stream or run outside it is not found (404). A source that is not an mcp
|
|
25
|
+
source: 409 ``not_mcp_source``; one without a valid recipe or a registered parser: 409
|
|
26
|
+
``source_not_ready``. Each ``POST`` counts against the token's write budget, ``next`` against its
|
|
27
|
+
reads (:mod:`memini.api.limits`).
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from collections.abc import Generator
|
|
31
|
+
from contextlib import contextmanager
|
|
32
|
+
from datetime import datetime
|
|
33
|
+
from typing import Any, Literal, cast
|
|
34
|
+
from uuid import UUID
|
|
35
|
+
|
|
36
|
+
from fastapi import APIRouter, Request
|
|
37
|
+
from pydantic import BaseModel, ConfigDict, Field, JsonValue
|
|
38
|
+
|
|
39
|
+
from memini import ingest
|
|
40
|
+
from memini.api.auth import IngestAccess
|
|
41
|
+
from memini.api.database import Connection
|
|
42
|
+
from memini.api.errors import ApiError
|
|
43
|
+
from memini.api.ingest import BatchRoute, ingest_errors
|
|
44
|
+
from memini.config import Settings
|
|
45
|
+
from memini.db import Pool
|
|
46
|
+
from memini.executors import agent
|
|
47
|
+
from memini.extraction.job import extraction_enabled
|
|
48
|
+
from memini.query import QueryServices
|
|
49
|
+
from memini.recipes import ParserError
|
|
50
|
+
from memini.responses import (
|
|
51
|
+
InvalidResponseError,
|
|
52
|
+
NotAnMcpSourceError,
|
|
53
|
+
ReplayError,
|
|
54
|
+
ResponseTooLargeError,
|
|
55
|
+
ServerInfo,
|
|
56
|
+
WindowClosedError,
|
|
57
|
+
WindowNotFoundError,
|
|
58
|
+
)
|
|
59
|
+
from memini.sources import SourceBusyError
|
|
60
|
+
from memini.workspaces import NotFoundError
|
|
61
|
+
|
|
62
|
+
__all__ = [
|
|
63
|
+
"AgentRunInfo",
|
|
64
|
+
"CallInfo",
|
|
65
|
+
"FinishAnswer",
|
|
66
|
+
"NextAnswer",
|
|
67
|
+
"ReceiptAnswer",
|
|
68
|
+
"ResponseBody",
|
|
69
|
+
"StartBody",
|
|
70
|
+
"WindowInfo",
|
|
71
|
+
"build_agent_runs_router",
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
_MAX_TEXT = 1_000_000
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# --- Bodies --------------------------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ServerBody(BaseModel):
|
|
81
|
+
"""What the session's server reported when it started (``serverInfo``)."""
|
|
82
|
+
|
|
83
|
+
model_config = ConfigDict(extra="forbid")
|
|
84
|
+
|
|
85
|
+
name: str = Field(min_length=1, max_length=256, description="serverInfo.name.")
|
|
86
|
+
version: str | None = Field(
|
|
87
|
+
default=None, min_length=1, max_length=128, description="serverInfo.version."
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class StartBody(BaseModel):
|
|
92
|
+
model_config = ConfigDict(extra="forbid")
|
|
93
|
+
|
|
94
|
+
stream: str = Field(
|
|
95
|
+
min_length=1, max_length=1024, description="The key of one of the source's streams."
|
|
96
|
+
)
|
|
97
|
+
server: ServerBody | None = Field(
|
|
98
|
+
default=None,
|
|
99
|
+
description="The session's server, if the agent can tell: another name than the recipe's"
|
|
100
|
+
" is refused. Left out: the recipe's server, version unknown.",
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class ResponseBody(BaseModel):
|
|
105
|
+
"""The raw result of a call handed out, as the agent's client got it."""
|
|
106
|
+
|
|
107
|
+
model_config = ConfigDict(extra="forbid")
|
|
108
|
+
|
|
109
|
+
call_id: UUID = Field(description="The id the call was handed out with.")
|
|
110
|
+
tool: str = Field(
|
|
111
|
+
min_length=1, max_length=128, description="The tool's name at the server, as handed out."
|
|
112
|
+
)
|
|
113
|
+
arguments: dict[str, Any] = Field(description="The arguments the call was made with.")
|
|
114
|
+
result: JsonValue = Field(
|
|
115
|
+
default=None,
|
|
116
|
+
description="The tool's result: an MCP result object (content, structuredContent,"
|
|
117
|
+
" isError), the list of its content blocks, or its text. Stored redacted, otherwise"
|
|
118
|
+
" verbatim.",
|
|
119
|
+
)
|
|
120
|
+
error: str | None = Field(
|
|
121
|
+
default=None,
|
|
122
|
+
max_length=_MAX_TEXT,
|
|
123
|
+
description="Instead of result: the error of a call that failed. The call is then handed"
|
|
124
|
+
" out again, a few times.",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# --- Answers -------------------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class WindowInfo(BaseModel):
|
|
132
|
+
position: int = Field(description="The window's place in the plan, from 0.")
|
|
133
|
+
kind: Literal["initial", "forward", "backfill"]
|
|
134
|
+
start: datetime = Field(serialization_alias="from")
|
|
135
|
+
end: datetime = Field(serialization_alias="to")
|
|
136
|
+
status: agent.WindowStatus
|
|
137
|
+
complete: bool | None = Field(description="Its claim once finish ingested it.")
|
|
138
|
+
|
|
139
|
+
@classmethod
|
|
140
|
+
def of(cls, window: agent.AgentWindow) -> "WindowInfo":
|
|
141
|
+
return cls(
|
|
142
|
+
position=window.position,
|
|
143
|
+
kind=window.kind,
|
|
144
|
+
start=window.start,
|
|
145
|
+
end=window.end,
|
|
146
|
+
status=window.status,
|
|
147
|
+
complete=window.complete,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class AgentRunInfo(BaseModel):
|
|
152
|
+
id: UUID
|
|
153
|
+
workspace: str
|
|
154
|
+
source: str
|
|
155
|
+
stream: str
|
|
156
|
+
state: Literal["open", "finishing", "finished"]
|
|
157
|
+
as_of: datetime = Field(description="The database time pinned for the run.")
|
|
158
|
+
server: str
|
|
159
|
+
server_version: str | None
|
|
160
|
+
recipe_digest: str
|
|
161
|
+
started_at: datetime
|
|
162
|
+
closed_at: datetime | None
|
|
163
|
+
finished_at: datetime | None
|
|
164
|
+
run_id: UUID | None = Field(description="The run report, once finished: GET /v1/runs/{id}.")
|
|
165
|
+
plan: dict[str, JsonValue]
|
|
166
|
+
windows: list[WindowInfo]
|
|
167
|
+
counts: dict[str, int] = Field(description="Calls handed out, responses stored, and so on.")
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def of(cls, run: agent.AgentRun) -> "AgentRunInfo":
|
|
171
|
+
return cls(
|
|
172
|
+
id=run.id,
|
|
173
|
+
workspace=run.workspace,
|
|
174
|
+
source=run.source,
|
|
175
|
+
stream=run.stream,
|
|
176
|
+
state=run.state,
|
|
177
|
+
as_of=run.as_of,
|
|
178
|
+
server=run.server.name,
|
|
179
|
+
server_version=run.server.version,
|
|
180
|
+
recipe_digest=run.recipe_digest,
|
|
181
|
+
started_at=run.started_at,
|
|
182
|
+
closed_at=run.closed_at,
|
|
183
|
+
finished_at=run.finished_at,
|
|
184
|
+
run_id=run.run_id,
|
|
185
|
+
plan=dict(run.plan),
|
|
186
|
+
windows=[WindowInfo.of(window) for window in run.windows],
|
|
187
|
+
counts=dict(run.counts),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class CallInfo(BaseModel):
|
|
192
|
+
id: UUID = Field(description="Post the call's response under this id.")
|
|
193
|
+
call: Literal["list", "detail"]
|
|
194
|
+
tool: str = Field(description="The tool's name at the server.")
|
|
195
|
+
arguments: dict[str, Any] = Field(description="Send exactly these, unchanged.")
|
|
196
|
+
page: int
|
|
197
|
+
item_id: str | None
|
|
198
|
+
check: str = Field(description="The completeness check the call makes pass.")
|
|
199
|
+
reason: str = Field(description="Why the call is needed, by position.")
|
|
200
|
+
attempt: int = Field(description="1 the first time this call of the window is handed out.")
|
|
201
|
+
|
|
202
|
+
@classmethod
|
|
203
|
+
def of(cls, call: agent.HandedOutCall) -> "CallInfo":
|
|
204
|
+
return cls(
|
|
205
|
+
id=call.id,
|
|
206
|
+
call=call.call,
|
|
207
|
+
tool=call.tool,
|
|
208
|
+
arguments=dict(call.arguments),
|
|
209
|
+
page=call.page,
|
|
210
|
+
item_id=call.item_id,
|
|
211
|
+
check=call.check,
|
|
212
|
+
reason=call.reason,
|
|
213
|
+
attempt=call.attempt,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class NextAnswer(BaseModel):
|
|
218
|
+
status: agent.NextStatus = Field(
|
|
219
|
+
description="call: make the call. window_done: ask again for the next window."
|
|
220
|
+
" run_done: finish the run."
|
|
221
|
+
)
|
|
222
|
+
call: CallInfo | None = None
|
|
223
|
+
window: WindowInfo | None = None
|
|
224
|
+
complete: bool | None = Field(
|
|
225
|
+
default=None,
|
|
226
|
+
description="window_done: whether the window passes every check, as far as the stored"
|
|
227
|
+
" responses tell before the parser runs.",
|
|
228
|
+
)
|
|
229
|
+
left: int = Field(
|
|
230
|
+
default=0, description="window_done: calls the window misses that are not handed out."
|
|
231
|
+
)
|
|
232
|
+
reason: str | None = Field(default=None, description="run_done: why.")
|
|
233
|
+
|
|
234
|
+
@classmethod
|
|
235
|
+
def of(cls, step: agent.NextStep) -> "NextAnswer":
|
|
236
|
+
return cls(
|
|
237
|
+
status=step.status,
|
|
238
|
+
call=None if step.call is None else CallInfo.of(step.call),
|
|
239
|
+
window=None if step.window is None else WindowInfo.of(step.window),
|
|
240
|
+
complete=step.complete,
|
|
241
|
+
left=step.left,
|
|
242
|
+
reason=step.reason,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class ReceiptAnswer(BaseModel):
|
|
247
|
+
"""What storing the response did; never a value of it."""
|
|
248
|
+
|
|
249
|
+
call_id: UUID
|
|
250
|
+
window: WindowInfo
|
|
251
|
+
position: int = Field(description="The response's place in its window, from 1.")
|
|
252
|
+
stored: bool = Field(description="False: the window held the same result for the call.")
|
|
253
|
+
is_error: bool = Field(description="The call failed; it is handed out again, a few times.")
|
|
254
|
+
responses: int = Field(description="Responses the window holds now.")
|
|
255
|
+
redacted: int = Field(description="Values redacted from the result and the arguments.")
|
|
256
|
+
|
|
257
|
+
@classmethod
|
|
258
|
+
def of(cls, receipt: agent.Receipt) -> "ReceiptAnswer":
|
|
259
|
+
return cls(
|
|
260
|
+
call_id=receipt.call_id,
|
|
261
|
+
window=WindowInfo.of(receipt.window),
|
|
262
|
+
position=receipt.position,
|
|
263
|
+
stored=receipt.stored,
|
|
264
|
+
is_error=receipt.is_error,
|
|
265
|
+
responses=receipt.responses,
|
|
266
|
+
redacted=receipt.redacted,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class WindowResult(BaseModel):
|
|
271
|
+
window: WindowInfo
|
|
272
|
+
ingested: bool = Field(description="False: never opened, skipped or gone.")
|
|
273
|
+
records: int = 0
|
|
274
|
+
inserted: int = 0
|
|
275
|
+
updated: int = 0
|
|
276
|
+
revived: int = 0
|
|
277
|
+
deleted: int = 0
|
|
278
|
+
unchanged: int = 0
|
|
279
|
+
reconciled: int = 0
|
|
280
|
+
checkpoint_moved: bool = False
|
|
281
|
+
incomplete: list[str] = Field(default_factory=list[str], description="Why it is incomplete.")
|
|
282
|
+
missing_calls: int = Field(default=0, description="Calls that would complete it.")
|
|
283
|
+
|
|
284
|
+
@classmethod
|
|
285
|
+
def of(cls, outcome: agent.WindowOutcome) -> "WindowResult":
|
|
286
|
+
done = outcome.ingest
|
|
287
|
+
info = WindowInfo.of(outcome.window)
|
|
288
|
+
if done is None:
|
|
289
|
+
return cls(window=info, ingested=False)
|
|
290
|
+
return cls(
|
|
291
|
+
window=info,
|
|
292
|
+
ingested=True,
|
|
293
|
+
records=done.records,
|
|
294
|
+
inserted=done.inserted,
|
|
295
|
+
updated=done.updated,
|
|
296
|
+
revived=done.revived,
|
|
297
|
+
deleted=done.deleted,
|
|
298
|
+
unchanged=done.unchanged,
|
|
299
|
+
reconciled=done.reconciled,
|
|
300
|
+
checkpoint_moved=done.checkpoint_moved,
|
|
301
|
+
incomplete=list(done.incomplete),
|
|
302
|
+
missing_calls=len(done.missing),
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class FinishAnswer(BaseModel):
|
|
307
|
+
agent_run: AgentRunInfo
|
|
308
|
+
run_id: UUID = Field(description="The run report: GET /v1/runs/{id}.")
|
|
309
|
+
windows: list[WindowResult]
|
|
310
|
+
checked_until: datetime | None = Field(description="The stream's checkpoint now.")
|
|
311
|
+
|
|
312
|
+
@classmethod
|
|
313
|
+
def of(cls, result: agent.FinishResult) -> "FinishAnswer":
|
|
314
|
+
return cls(
|
|
315
|
+
agent_run=AgentRunInfo.of(result.agent_run),
|
|
316
|
+
run_id=result.run_id,
|
|
317
|
+
windows=[WindowResult.of(outcome) for outcome in result.windows],
|
|
318
|
+
checked_until=result.checked_until,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# --- Errors --------------------------------------------------------------------------------------
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@contextmanager
|
|
326
|
+
def _agent_errors() -> Generator[None]:
|
|
327
|
+
try:
|
|
328
|
+
yield
|
|
329
|
+
except NotFoundError as error:
|
|
330
|
+
raise ApiError(404, "not_found", f"No such {error.what}.") from None
|
|
331
|
+
except NotAnMcpSourceError:
|
|
332
|
+
raise ApiError(
|
|
333
|
+
409, "not_mcp_source", "Agent runs are for mcp sources; this source is not one."
|
|
334
|
+
) from None
|
|
335
|
+
except agent.SourceNotReadyError as error:
|
|
336
|
+
raise ApiError(409, "source_not_ready", f"The source cannot run: {error}.") from None
|
|
337
|
+
except agent.ServerMismatchError as error:
|
|
338
|
+
raise ApiError(409, "server_mismatch", f"{_sentence(error)}.") from None
|
|
339
|
+
except agent.AgentRunOpenError as error:
|
|
340
|
+
raise ApiError(409, "agent_run_open", f"{_sentence(error)}.") from None
|
|
341
|
+
except agent.AgentRunClosedError as error:
|
|
342
|
+
code = "agent_run_finished" if error.finished else "agent_run_finishing"
|
|
343
|
+
raise ApiError(409, code, f"{_sentence(error)}.") from None
|
|
344
|
+
except agent.RecipeChangedError as error:
|
|
345
|
+
raise ApiError(409, "recipe_changed", f"{_sentence(error)}.") from None
|
|
346
|
+
except agent.CallNotHandedOutError as error:
|
|
347
|
+
raise ApiError(409, "call_not_handed_out", f"{_sentence(error)}; nothing stored.") from None
|
|
348
|
+
except agent.CallMismatchError as error:
|
|
349
|
+
raise ApiError(409, "call_mismatch", f"{_sentence(error)}; nothing stored.") from None
|
|
350
|
+
except agent.InvalidResultError as error:
|
|
351
|
+
raise ApiError(422, "invalid_result", f"{_sentence(error)}.") from None
|
|
352
|
+
except ResponseTooLargeError as error:
|
|
353
|
+
raise ApiError(413, "response_too_large", f"The response is too large: {error}.") from None
|
|
354
|
+
except InvalidResponseError as error:
|
|
355
|
+
raise ApiError(
|
|
356
|
+
422, "invalid_response", f"The response cannot be stored: {error}."
|
|
357
|
+
) from None
|
|
358
|
+
except WindowNotFoundError:
|
|
359
|
+
raise ApiError(
|
|
360
|
+
409, "window_gone", "The call's window was deleted; finish the run and start another."
|
|
361
|
+
) from None
|
|
362
|
+
except WindowClosedError:
|
|
363
|
+
raise ApiError(409, "window_closed", "The call's window was parsed already.") from None
|
|
364
|
+
except SourceBusyError as error:
|
|
365
|
+
raise ApiError(409, "source_busy", f"{_sentence(error)}.") from None
|
|
366
|
+
except ReplayError as error:
|
|
367
|
+
raise ApiError(409, "recipe_mismatch", f"{_sentence(error)}.") from None
|
|
368
|
+
except ParserError as error:
|
|
369
|
+
raise ApiError(409, "parser_failed", f"The parser failed: {error}.") from None
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _sentence(error: Exception) -> str:
|
|
373
|
+
text = str(error)
|
|
374
|
+
return text[:1].upper() + text[1:]
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# --- Routes --------------------------------------------------------------------------------------
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def build_agent_runs_router() -> APIRouter:
|
|
381
|
+
"""The agent executor's routes; see the module."""
|
|
382
|
+
router = APIRouter(
|
|
383
|
+
prefix="/workspaces/{workspace}/sources/{source}/agent-runs",
|
|
384
|
+
tags=["agent runs"],
|
|
385
|
+
route_class=BatchRoute,
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
@router.post("", status_code=201, summary="Start an agent run of a stream")
|
|
389
|
+
async def start_agent_run(
|
|
390
|
+
workspace: str, source: str, body: StartBody, access: IngestAccess, conn: Connection
|
|
391
|
+
) -> AgentRunInfo:
|
|
392
|
+
"""Pin the database time, plan the stream's windows and open the first one."""
|
|
393
|
+
server = None if body.server is None else ServerInfo(body.server.name, body.server.version)
|
|
394
|
+
with _agent_errors():
|
|
395
|
+
run = await agent.start_run(conn, access, workspace, source, body.stream, server=server)
|
|
396
|
+
return AgentRunInfo.of(run)
|
|
397
|
+
|
|
398
|
+
@router.get("/{run_id}", summary="Read an agent run")
|
|
399
|
+
async def read_agent_run(
|
|
400
|
+
workspace: str, source: str, run_id: UUID, access: IngestAccess, conn: Connection
|
|
401
|
+
) -> AgentRunInfo:
|
|
402
|
+
with _agent_errors():
|
|
403
|
+
run = await agent.get_agent_run(conn, access, workspace, source, run_id)
|
|
404
|
+
return AgentRunInfo.of(run)
|
|
405
|
+
|
|
406
|
+
@router.get("/{run_id}/next", summary="The next call of an agent run")
|
|
407
|
+
async def next_agent_call(
|
|
408
|
+
workspace: str, source: str, run_id: UUID, access: IngestAccess, conn: Connection
|
|
409
|
+
) -> NextAnswer:
|
|
410
|
+
"""The next call to make, or that the window or the run is done."""
|
|
411
|
+
with _agent_errors():
|
|
412
|
+
step = await agent.next_call(conn, access, workspace, source, run_id)
|
|
413
|
+
return NextAnswer.of(step)
|
|
414
|
+
|
|
415
|
+
@router.post("/{run_id}/responses", summary="Store the raw response of a call handed out")
|
|
416
|
+
async def store_agent_response(
|
|
417
|
+
workspace: str,
|
|
418
|
+
source: str,
|
|
419
|
+
run_id: UUID,
|
|
420
|
+
body: ResponseBody,
|
|
421
|
+
access: IngestAccess,
|
|
422
|
+
conn: Connection,
|
|
423
|
+
) -> ReceiptAnswer:
|
|
424
|
+
"""Store the result of a call this run handed out, redacted and otherwise verbatim. The
|
|
425
|
+
answer counts; it never repeats the response."""
|
|
426
|
+
with _agent_errors():
|
|
427
|
+
receipt = await agent.submit_response(
|
|
428
|
+
conn,
|
|
429
|
+
access,
|
|
430
|
+
workspace,
|
|
431
|
+
source,
|
|
432
|
+
run_id,
|
|
433
|
+
body.call_id,
|
|
434
|
+
tool=body.tool,
|
|
435
|
+
arguments=body.arguments,
|
|
436
|
+
result=body.result,
|
|
437
|
+
error=body.error,
|
|
438
|
+
)
|
|
439
|
+
return ReceiptAnswer.of(receipt)
|
|
440
|
+
|
|
441
|
+
@router.post("/{run_id}/finish", summary="Finish an agent run")
|
|
442
|
+
async def finish_agent_run(
|
|
443
|
+
workspace: str,
|
|
444
|
+
source: str,
|
|
445
|
+
run_id: UUID,
|
|
446
|
+
access: IngestAccess,
|
|
447
|
+
request: Request,
|
|
448
|
+
conn: Connection,
|
|
449
|
+
) -> FinishAnswer:
|
|
450
|
+
"""Close the run to calls, parse and ingest its windows, write the run report."""
|
|
451
|
+
settings = cast(Settings, request.app.state.settings)
|
|
452
|
+
pool = cast(Pool, request.app.state.pool)
|
|
453
|
+
embedder = cast(QueryServices, request.app.state.services).embedder
|
|
454
|
+
indexer = None if embedder is None else ingest.PayloadIndexer(embedder)
|
|
455
|
+
with _agent_errors(), ingest_errors():
|
|
456
|
+
result = await agent.finish_run(
|
|
457
|
+
conn,
|
|
458
|
+
access,
|
|
459
|
+
workspace,
|
|
460
|
+
source,
|
|
461
|
+
run_id,
|
|
462
|
+
runs=pool,
|
|
463
|
+
limits=ingest.IngestLimits.from_settings(settings),
|
|
464
|
+
indexer=indexer,
|
|
465
|
+
extract=extraction_enabled(settings),
|
|
466
|
+
)
|
|
467
|
+
return FinishAnswer.of(result)
|
|
468
|
+
|
|
469
|
+
return router
|