tai42-e2e 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tai42_e2e/__init__.py +19 -0
- tai42_e2e/_threaded.py +44 -0
- tai42_e2e/booting.py +73 -0
- tai42_e2e/diagnostics.py +67 -0
- tai42_e2e/harness.py +466 -0
- tai42_e2e/httpapi.py +151 -0
- tai42_e2e/llmstub.py +278 -0
- tai42_e2e/manifests.py +1583 -0
- tai42_e2e/marketplace.py +535 -0
- tai42_e2e/mcp.py +89 -0
- tai42_e2e/metrics.py +45 -0
- tai42_e2e/netfixtures.py +1101 -0
- tai42_e2e/pg.py +107 -0
- tai42_e2e/pkgsource.py +379 -0
- tai42_e2e/ports.py +58 -0
- tai42_e2e/procs.py +162 -0
- tai42_e2e/py.typed +0 -0
- tai42_e2e/pytest_plugin.py +156 -0
- tai42_e2e/rabbitx.py +87 -0
- tai42_e2e/redisx.py +103 -0
- tai42_e2e/settings.py +90 -0
- tai42_e2e/stack.py +951 -0
- tai42_e2e/studio_runner.py +516 -0
- tai42_e2e/tcprelay.py +265 -0
- tai42_e2e/variants.py +468 -0
- tai42_e2e/waiting.py +78 -0
- tai42_e2e-0.3.0.dist-info/METADATA +234 -0
- tai42_e2e-0.3.0.dist-info/RECORD +43 -0
- tai42_e2e-0.3.0.dist-info/WHEEL +5 -0
- tai42_e2e-0.3.0.dist-info/entry_points.txt +5 -0
- tai42_e2e-0.3.0.dist-info/licenses/LICENSE +202 -0
- tai42_e2e-0.3.0.dist-info/licenses/NOTICE +9 -0
- tai42_e2e-0.3.0.dist-info/top_level.txt +2 -0
- tai42_e2e_fixtures/__init__.py +4 -0
- tai42_e2e_fixtures/connector_provider.py +161 -0
- tai42_e2e_fixtures/embed_main.py +57 -0
- tai42_e2e_fixtures/identity_provider.py +176 -0
- tai42_e2e_fixtures/managed_mcp_server.py +55 -0
- tai42_e2e_fixtures/monitor_backend.py +71 -0
- tai42_e2e_fixtures/py.typed +0 -0
- tai42_e2e_fixtures/storage.py +129 -0
- tai42_e2e_fixtures/stub_channel.py +35 -0
- tai42_e2e_fixtures/tools.py +176 -0
tai42_e2e/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""tai42-e2e harness: boot the real multi-process tai deployment topology and
|
|
2
|
+
drive it over HTTP. Imported by the tests, never by the system under test."""
|
|
3
|
+
|
|
4
|
+
from tai42_e2e.settings import HarnessSettings
|
|
5
|
+
from tai42_e2e.stack import Infra, StackConfig, StackResources, TaiStack, Topology
|
|
6
|
+
from tai42_e2e.waiting import WaitTimeout, align_to_window, wait_for, wait_for_async
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"HarnessSettings",
|
|
10
|
+
"Infra",
|
|
11
|
+
"StackConfig",
|
|
12
|
+
"StackResources",
|
|
13
|
+
"TaiStack",
|
|
14
|
+
"Topology",
|
|
15
|
+
"WaitTimeout",
|
|
16
|
+
"align_to_window",
|
|
17
|
+
"wait_for",
|
|
18
|
+
"wait_for_async",
|
|
19
|
+
]
|
tai42_e2e/_threaded.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Run a uvicorn app in a background thread of the pytest process.
|
|
2
|
+
|
|
3
|
+
The LLM stub and the net fixtures are harness infrastructure, not the system
|
|
4
|
+
under test, so they live in-process on allocated ports rather than as spawned
|
|
5
|
+
processes or compose services."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import threading
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import uvicorn
|
|
13
|
+
|
|
14
|
+
from tai42_e2e.waiting import wait_for
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ThreadedServer:
|
|
18
|
+
"""A uvicorn server bound to an ASGI app, started/stopped from a thread."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, app: Any, host: str, port: int) -> None:
|
|
21
|
+
config = uvicorn.Config(app, host=host, port=port, log_level="warning", lifespan="off")
|
|
22
|
+
self._server = uvicorn.Server(config)
|
|
23
|
+
self._thread = threading.Thread(target=self._server.run, daemon=True)
|
|
24
|
+
self.host = host
|
|
25
|
+
self.port = port
|
|
26
|
+
|
|
27
|
+
def start(self) -> None:
|
|
28
|
+
self._thread.start()
|
|
29
|
+
wait_for(
|
|
30
|
+
lambda: self._server.started,
|
|
31
|
+
deadline=10.0,
|
|
32
|
+
message=f"threaded server never started on {self.host}:{self.port}",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
def stop(self) -> None:
|
|
36
|
+
self._server.should_exit = True
|
|
37
|
+
self._thread.join(timeout=10.0)
|
|
38
|
+
|
|
39
|
+
def __enter__(self) -> ThreadedServer:
|
|
40
|
+
self.start()
|
|
41
|
+
return self
|
|
42
|
+
|
|
43
|
+
def __exit__(self, *exc: object) -> None:
|
|
44
|
+
self.stop()
|
tai42_e2e/booting.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Stack allocate-build-boot orchestration shared by the pytest fixtures.
|
|
2
|
+
|
|
3
|
+
The profile fixtures (in ``tests/conftest.py`` and the per-suite conftests) call
|
|
4
|
+
:func:`boot_stack` to turn a manifest builder into a live, isolated
|
|
5
|
+
:class:`~tai42_e2e.stack.TaiStack` for the duration of a fixture scope. Keeping the
|
|
6
|
+
orchestration in the harness library — rather than a conftest-local helper — lets
|
|
7
|
+
a per-suite conftest reuse it without importing another conftest module (two
|
|
8
|
+
conftests share the same basename, so a cross-conftest import is ambiguous)."""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from tai42_e2e import diagnostics
|
|
17
|
+
from tai42_e2e.harness import allocate_resources, release_resources, seed_bootstrap_key
|
|
18
|
+
from tai42_e2e.stack import Infra, StackConfig, StackResources, TaiStack
|
|
19
|
+
from tai42_e2e.variants import Variants
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def allocate_and_build(
|
|
23
|
+
infra: Infra,
|
|
24
|
+
root: Path,
|
|
25
|
+
builder: Callable[[StackResources, Variants], StackConfig],
|
|
26
|
+
resource_kwargs: Mapping[str, Any] | None,
|
|
27
|
+
allocate_checkpoint_db: bool,
|
|
28
|
+
) -> tuple[StackResources, StackConfig]:
|
|
29
|
+
"""Allocate a stack's resources and render its config. A builder that raises
|
|
30
|
+
(a profile whose prerequisite is missing) reaps the allocation instead of
|
|
31
|
+
leaking the Redis index, the Postgres clone and the broker lease for the rest
|
|
32
|
+
of the session."""
|
|
33
|
+
resources = allocate_resources(
|
|
34
|
+
infra, root, allocate_checkpoint_db=allocate_checkpoint_db, **(resource_kwargs or {})
|
|
35
|
+
)
|
|
36
|
+
try:
|
|
37
|
+
return resources, builder(resources, infra.variants)
|
|
38
|
+
except BaseException:
|
|
39
|
+
release_resources(infra, resources)
|
|
40
|
+
raise
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def boot_stack(
|
|
44
|
+
infra: Infra,
|
|
45
|
+
root: Path,
|
|
46
|
+
builder: Callable[[StackResources, Variants], StackConfig],
|
|
47
|
+
*,
|
|
48
|
+
resource_kwargs: Mapping[str, Any] | None = None,
|
|
49
|
+
seed_auth: bool = False,
|
|
50
|
+
allocate_checkpoint_db: bool = False,
|
|
51
|
+
) -> Iterator[TaiStack]:
|
|
52
|
+
"""Allocate, build, boot, and yield a stack; tear it down at scope end. A
|
|
53
|
+
generator so a fixture ``yield from`` s it."""
|
|
54
|
+
resources, config = allocate_and_build(infra, root, builder, resource_kwargs, allocate_checkpoint_db)
|
|
55
|
+
# Seed the auth bootstrap (Redis identity/context + PG policy/route rows)
|
|
56
|
+
# BEFORE the stack boots: readiness probes /health and /metrics, which are
|
|
57
|
+
# denied (403) under access control until the route table pins them public,
|
|
58
|
+
# so the seed has to land before the processes answer their first request.
|
|
59
|
+
stack = TaiStack(config, infra, resources, root)
|
|
60
|
+
try:
|
|
61
|
+
# The auth seed runs before boot (readiness probes are denied until the
|
|
62
|
+
# route table pins them public), so a seed failure would otherwise leak
|
|
63
|
+
# the resources allocated above; reap them on failure.
|
|
64
|
+
auth_token = seed_bootstrap_key(infra, resources) if seed_auth else None
|
|
65
|
+
except BaseException:
|
|
66
|
+
stack.teardown()
|
|
67
|
+
raise
|
|
68
|
+
# Set the token BEFORE boot: readiness drains the boot reload gate through the
|
|
69
|
+
# MCP probe, which an access-controlled stack fences, so the probe must carry the
|
|
70
|
+
# seeded root token that boot's own readiness runs under.
|
|
71
|
+
stack.auth_token = auth_token
|
|
72
|
+
with stack, diagnostics.track(stack):
|
|
73
|
+
yield stack
|
tai42_e2e/diagnostics.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Failure diagnostics: on a failed test that used a stack, attach the tail of
|
|
2
|
+
each process log, the rendered manifest, the masked env map, and the port table
|
|
3
|
+
so the failure is actionable from a CI page without a re-run."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import contextlib
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from tai42_e2e.stack import TaiStack
|
|
13
|
+
|
|
14
|
+
# Stacks currently live within a running test, tracked so the failure hook can
|
|
15
|
+
# find them without threading the fixture through every assertion.
|
|
16
|
+
_active: list[TaiStack] = []
|
|
17
|
+
|
|
18
|
+
_SECRET_HINTS = ("secret", "password", "kek", "hmac", "api_key", "token")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def register(stack: TaiStack) -> None:
|
|
22
|
+
"""Mark ``stack`` active so the failure hook can dump it."""
|
|
23
|
+
_active.append(stack)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def unregister(stack: TaiStack) -> None:
|
|
27
|
+
"""Drop ``stack`` from the active set (idempotent)."""
|
|
28
|
+
with contextlib.suppress(ValueError):
|
|
29
|
+
_active.remove(stack)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@contextlib.contextmanager
|
|
33
|
+
def track(stack: TaiStack) -> Iterator[TaiStack]:
|
|
34
|
+
"""Register ``stack`` as active for the duration of a test."""
|
|
35
|
+
register(stack)
|
|
36
|
+
try:
|
|
37
|
+
yield stack
|
|
38
|
+
finally:
|
|
39
|
+
unregister(stack)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _mask(key: str, value: str) -> str:
|
|
43
|
+
lowered = key.lower()
|
|
44
|
+
if any(hint in lowered for hint in _SECRET_HINTS):
|
|
45
|
+
return "***"
|
|
46
|
+
return value
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _stack_report(stack: TaiStack) -> str:
|
|
50
|
+
lines = [f"=== stack {stack.config.name!r} ({stack.config.topology.value}) ==="]
|
|
51
|
+
lines.append(f"app_ports={stack.app_ports} metrics_port={stack.metrics_port} metrics_dir={stack.metrics_dir}")
|
|
52
|
+
lines.append("env:")
|
|
53
|
+
for key, value in sorted(stack.config.env.items()):
|
|
54
|
+
lines.append(f" {key}={_mask(key, value)}")
|
|
55
|
+
lines.append("manifest:")
|
|
56
|
+
lines.append(" " + repr(stack.config.manifest))
|
|
57
|
+
for name, handle in stack._procs.items():
|
|
58
|
+
lines.append(f"--- {name} log tail ---")
|
|
59
|
+
lines.append(handle.log_tail())
|
|
60
|
+
return "\n".join(lines)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def report() -> str:
|
|
64
|
+
"""A diagnostic dump for every currently-active stack."""
|
|
65
|
+
if not _active:
|
|
66
|
+
return ""
|
|
67
|
+
return "\n\n".join(_stack_report(stack) for stack in _active)
|
tai42_e2e/harness.py
ADDED
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
"""Reusable stack orchestration: infra connect, per-stack resource allocation,
|
|
2
|
+
and the access-control bootstrap seed.
|
|
3
|
+
|
|
4
|
+
These live in the installable package (not in ``tests/conftest.py``) so BOTH the
|
|
5
|
+
pytest suite and the standalone ``tai42-e2e-studio-stack`` console runner drive one
|
|
6
|
+
implementation. The pytest fixtures wrap these; the runner calls them directly.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import secrets
|
|
13
|
+
import uuid
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import psycopg
|
|
18
|
+
|
|
19
|
+
from tai42_e2e.pg import PostgresAdmin
|
|
20
|
+
from tai42_e2e.redisx import RedisAdmin
|
|
21
|
+
from tai42_e2e.settings import HarnessSettings
|
|
22
|
+
from tai42_e2e.stack import Infra, InfraUnavailable, StackResources
|
|
23
|
+
from tai42_e2e.variants import resolve_variants
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def connect_infra(settings: HarnessSettings) -> Infra:
|
|
27
|
+
"""Resolve the variant set, connect the Redis + Postgres admin clients,
|
|
28
|
+
verify everything the selected variants need is reachable (loudly, with the
|
|
29
|
+
compose hint on failure), apply the DDL template, and return the
|
|
30
|
+
:class:`Infra` bundle. The caller owns closing ``infra.redis``."""
|
|
31
|
+
variants = resolve_variants(settings)
|
|
32
|
+
host, port = settings.redis_host_port
|
|
33
|
+
redis_admin = RedisAdmin(host, port)
|
|
34
|
+
pg_admin = PostgresAdmin(settings)
|
|
35
|
+
try:
|
|
36
|
+
redis_admin.check_reachable()
|
|
37
|
+
except Exception as exc:
|
|
38
|
+
raise InfraUnavailable(f"Redis not usable ({exc}). Start it with `docker compose up -d`.") from exc
|
|
39
|
+
try:
|
|
40
|
+
pg_admin.check_reachable()
|
|
41
|
+
except Exception as exc:
|
|
42
|
+
raise InfraUnavailable(f"Postgres not reachable ({exc}). Start it with `docker compose up -d`.") from exc
|
|
43
|
+
# Extra reachability the backend needs beyond the shared Redis + Postgres
|
|
44
|
+
# (celery's broker); a no-op for the Redis-only backends.
|
|
45
|
+
variants.backend.infra_check(settings)
|
|
46
|
+
# The module-capable checkpoint Redis is optional (present only when its URL is set);
|
|
47
|
+
# when set it must be reachable, failing loudly like the main Redis. Its allocator owns
|
|
48
|
+
# DB 0 ONLY: RediSearch refuses indexes on db != 0 and index names are server-global,
|
|
49
|
+
# so a stack on DB 1 would silently read another stack's index. A one-slot pool makes a
|
|
50
|
+
# second concurrent checkpoint stack fail loudly instead of mis-indexing; FLUSHDB at
|
|
51
|
+
# allocation drops the indexes so successive stacks reuse DB 0 cleanly.
|
|
52
|
+
checkpoint_redis: RedisAdmin | None = None
|
|
53
|
+
if settings.checkpoint_redis_url is not None:
|
|
54
|
+
ck_host, ck_port = settings.checkpoint_redis_host_port
|
|
55
|
+
checkpoint_redis = RedisAdmin(ck_host, ck_port, stack_db_range=range(1))
|
|
56
|
+
try:
|
|
57
|
+
checkpoint_redis.check_reachable()
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
raise InfraUnavailable(
|
|
60
|
+
f"Checkpoint Redis not reachable ({exc}). Start it with `docker compose --profile agents-redis up -d`."
|
|
61
|
+
) from exc
|
|
62
|
+
try:
|
|
63
|
+
# The checkpoint Redis exists only for its modules; a module-free image
|
|
64
|
+
# here is a misconfiguration, caught loudly at session start.
|
|
65
|
+
checkpoint_redis.check_search_json_modules()
|
|
66
|
+
except Exception as exc:
|
|
67
|
+
raise InfraUnavailable(
|
|
68
|
+
f"Checkpoint Redis lacks the RediSearch/RedisJSON modules the langgraph redis provider needs "
|
|
69
|
+
f"({exc}). Use a module-capable image (redis:8); `docker compose --profile agents-redis up -d`."
|
|
70
|
+
) from exc
|
|
71
|
+
pg_admin.ensure_template()
|
|
72
|
+
return Infra(
|
|
73
|
+
settings=settings, redis=redis_admin, pg=pg_admin, variants=variants, checkpoint_redis=checkpoint_redis
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def allocate_resources(
|
|
78
|
+
infra: Infra,
|
|
79
|
+
root: Path,
|
|
80
|
+
*,
|
|
81
|
+
allocate_checkpoint_db: bool = False,
|
|
82
|
+
redis_host: str | None = None,
|
|
83
|
+
redis_port: int | None = None,
|
|
84
|
+
bus_redis_host: str | None = None,
|
|
85
|
+
bus_redis_port: int | None = None,
|
|
86
|
+
pg_host: str | None = None,
|
|
87
|
+
pg_port: int | None = None,
|
|
88
|
+
**extra: object,
|
|
89
|
+
) -> StackResources:
|
|
90
|
+
"""Reserve a Redis logical DB, a per-stack Postgres database clone, and (for
|
|
91
|
+
a broker-bearing backend) a per-stack broker lease, then pack the coordinates
|
|
92
|
+
a manifest/env builder needs.
|
|
93
|
+
|
|
94
|
+
``allocate_checkpoint_db`` additionally reserves a logical DB on the
|
|
95
|
+
module-capable checkpoint Redis (the langgraph redis checkpoint/store leg);
|
|
96
|
+
it requires the checkpoint Redis to be configured (``connect_infra`` built
|
|
97
|
+
it), raising loudly otherwise so a mis-gated fixture never silently runs the
|
|
98
|
+
memory provider.
|
|
99
|
+
|
|
100
|
+
``redis_host``/``redis_port`` and ``pg_host``/``pg_port`` point the SUT at
|
|
101
|
+
DIFFERENT connection ENDPOINTS for the stores the harness still allocates and
|
|
102
|
+
seeds on the real infra: the logical Redis DB and the Postgres database clone
|
|
103
|
+
are created as always, and the returned resources simply reach them through
|
|
104
|
+
the given endpoints — the infra-outage specs point them at a per-stack TCP
|
|
105
|
+
relay they can sever. The Redis override is an endpoint, not a URL, because
|
|
106
|
+
the logical DB index is chosen here: the returned ``redis_url`` always selects
|
|
107
|
+
the index this call reserved, so ``redis_url`` and ``redis_idx`` can never
|
|
108
|
+
disagree. ``probe_redis_url`` is deliberately NOT rerouted — it stays on the
|
|
109
|
+
real Redis so the harness can still read probe records while the SUT's own
|
|
110
|
+
endpoint is severed.
|
|
111
|
+
|
|
112
|
+
``bus_redis_host``/``bus_redis_port`` reroute the app worker bus through its OWN
|
|
113
|
+
endpoint, INDEPENDENT of the feature-Redis one, so a bus-outage spec severs the
|
|
114
|
+
bus (its own relay) without severing auth/feature stores. Unset, the bus rides
|
|
115
|
+
the real Redis directly. The returned ``bus_redis_url`` selects the stack's own
|
|
116
|
+
logical DB (presence keys flush with the DB on release); ``bus_namespace`` is
|
|
117
|
+
unique per stack because bus pub/sub + presence keys are server-global and the
|
|
118
|
+
logical-DB isolation does NOT isolate them."""
|
|
119
|
+
# No TaiStack exists yet to reap these on failure, so if a later step raises, release
|
|
120
|
+
# each already-acquired resource here — a partial allocation must not leak. Each cleanup
|
|
121
|
+
# runs only for a resource actually acquired.
|
|
122
|
+
if allocate_checkpoint_db and infra.checkpoint_redis is None:
|
|
123
|
+
raise RuntimeError(
|
|
124
|
+
"allocate_checkpoint_db=True but the checkpoint Redis is not configured "
|
|
125
|
+
"(set TAI_E2E_CHECKPOINT_REDIS_URL and start `docker compose --profile agents-redis up -d`)"
|
|
126
|
+
)
|
|
127
|
+
idx = infra.redis.allocate_db()
|
|
128
|
+
checkpoint_idx: int | None = None
|
|
129
|
+
lease = None
|
|
130
|
+
dbname: str | None = None
|
|
131
|
+
try:
|
|
132
|
+
if allocate_checkpoint_db:
|
|
133
|
+
assert infra.checkpoint_redis is not None
|
|
134
|
+
checkpoint_idx = infra.checkpoint_redis.allocate_db()
|
|
135
|
+
stack_id = uuid.uuid4().hex[:6]
|
|
136
|
+
dbname = f"tai42_e2e_{stack_id}"
|
|
137
|
+
infra.pg.create_stack_db(dbname)
|
|
138
|
+
storage_root = root / "storage"
|
|
139
|
+
storage_root.mkdir(parents=True, exist_ok=True)
|
|
140
|
+
lease = infra.variants.backend.allocate_broker(infra, stack_id)
|
|
141
|
+
settings = infra.settings
|
|
142
|
+
checkpoint_url = (
|
|
143
|
+
infra.checkpoint_redis.url_for(checkpoint_idx)
|
|
144
|
+
if checkpoint_idx is not None and infra.checkpoint_redis is not None
|
|
145
|
+
else None
|
|
146
|
+
)
|
|
147
|
+
infra_redis_host, infra_redis_port = settings.redis_host_port
|
|
148
|
+
sut_redis_host = redis_host if redis_host is not None else infra_redis_host
|
|
149
|
+
sut_redis_port = redis_port if redis_port is not None else infra_redis_port
|
|
150
|
+
sut_bus_host = bus_redis_host if bus_redis_host is not None else infra_redis_host
|
|
151
|
+
sut_bus_port = bus_redis_port if bus_redis_port is not None else infra_redis_port
|
|
152
|
+
# The bus namespace reuses the per-stack id (``dbname`` is ``tai42_e2e_<id>``):
|
|
153
|
+
# a single unique-per-stack token isolates the server-global bus channels.
|
|
154
|
+
# Inside the reap guard: an unknown key in ``extra`` raises TypeError here, and
|
|
155
|
+
# that must not strand the DB clone, the vhost lease and the logical indices.
|
|
156
|
+
return StackResources(
|
|
157
|
+
redis_idx=idx,
|
|
158
|
+
redis_url=f"redis://{sut_redis_host}:{sut_redis_port}/{idx}",
|
|
159
|
+
probe_redis_url=infra.redis.url_for(0),
|
|
160
|
+
bus_redis_url=f"redis://{sut_bus_host}:{sut_bus_port}/{idx}",
|
|
161
|
+
bus_namespace=dbname,
|
|
162
|
+
pg_host=pg_host if pg_host is not None else settings.pg_host,
|
|
163
|
+
pg_port=pg_port if pg_port is not None else settings.pg_port,
|
|
164
|
+
pg_user=settings.pg_user,
|
|
165
|
+
pg_password=settings.pg_password,
|
|
166
|
+
pg_db=dbname,
|
|
167
|
+
storage_root=str(storage_root),
|
|
168
|
+
broker_url=lease.broker_url if lease is not None else None,
|
|
169
|
+
broker_lease=lease,
|
|
170
|
+
checkpoint_redis_idx=checkpoint_idx,
|
|
171
|
+
checkpoint_redis_url=checkpoint_url,
|
|
172
|
+
**extra, # type: ignore[arg-type]
|
|
173
|
+
)
|
|
174
|
+
except BaseException:
|
|
175
|
+
if lease is not None:
|
|
176
|
+
lease.release()
|
|
177
|
+
if dbname is not None:
|
|
178
|
+
infra.pg.drop_stack_db(dbname)
|
|
179
|
+
if checkpoint_idx is not None and infra.checkpoint_redis is not None:
|
|
180
|
+
infra.checkpoint_redis.release_db(checkpoint_idx)
|
|
181
|
+
infra.redis.release_db(idx)
|
|
182
|
+
raise
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def release_resources(infra: Infra, resources: StackResources) -> None:
|
|
186
|
+
"""Reap an allocation that no :class:`TaiStack` owns yet — the mirror of
|
|
187
|
+
``TaiStack.teardown``'s resource release, for a failure between
|
|
188
|
+
``allocate_resources`` and the stack that would otherwise reap it. Every resource
|
|
189
|
+
is released even if one release raises (a flaky vhost delete must not strand the
|
|
190
|
+
Postgres clone and the Redis index); failures are collected and raised together so
|
|
191
|
+
they surface without displacing the error that led here (they chain under it)."""
|
|
192
|
+
errors: list[str] = []
|
|
193
|
+
if resources.broker_lease is not None:
|
|
194
|
+
try:
|
|
195
|
+
resources.broker_lease.release()
|
|
196
|
+
except Exception as exc:
|
|
197
|
+
errors.append(f"release broker lease: {exc!r}")
|
|
198
|
+
try:
|
|
199
|
+
infra.pg.drop_stack_db(resources.pg_db)
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
errors.append(f"drop stack db {resources.pg_db}: {exc!r}")
|
|
202
|
+
if resources.checkpoint_redis_idx is not None:
|
|
203
|
+
if infra.checkpoint_redis is None:
|
|
204
|
+
errors.append("resources hold a checkpoint Redis DB but infra.checkpoint_redis is None (leak)")
|
|
205
|
+
else:
|
|
206
|
+
infra.checkpoint_redis.release_db(resources.checkpoint_redis_idx)
|
|
207
|
+
infra.redis.release_db(resources.redis_idx)
|
|
208
|
+
if errors:
|
|
209
|
+
raise RuntimeError("releasing an unowned allocation found leaks:\n " + "\n ".join(errors))
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ---- access-control bootstrap seed --------------------------------------
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def seed_root_identity(
|
|
216
|
+
infra: Infra,
|
|
217
|
+
resources: StackResources,
|
|
218
|
+
*,
|
|
219
|
+
user_id: str,
|
|
220
|
+
scopes: Sequence[str],
|
|
221
|
+
raw: str | None = None,
|
|
222
|
+
) -> str:
|
|
223
|
+
"""Seed ONE key across the two storage homes the mint orchestration writes:
|
|
224
|
+
the identity provider's private record (delegated to the identity variant,
|
|
225
|
+
since that wire format is the provider's storage, not a harness contract) and
|
|
226
|
+
a policy row in the Postgres policy store. ``raw`` pins the token (the
|
|
227
|
+
browser-e2e runner needs a known key up front); left unset it is minted.
|
|
228
|
+
Returns the raw ``sk-...`` token the caller authenticates with."""
|
|
229
|
+
raw = raw if raw is not None else f"sk-{secrets.token_urlsafe(32)}"
|
|
230
|
+
hashed = hashlib.sha256(raw.encode()).hexdigest()
|
|
231
|
+
scope_list = list(scopes)
|
|
232
|
+
|
|
233
|
+
infra.variants.identity.seed_identity(infra, resources, user_id=user_id, hashed=hashed)
|
|
234
|
+
|
|
235
|
+
with psycopg.connect(
|
|
236
|
+
host=resources.pg_host,
|
|
237
|
+
port=resources.pg_port,
|
|
238
|
+
user=resources.pg_user,
|
|
239
|
+
password=resources.pg_password,
|
|
240
|
+
dbname=resources.pg_db,
|
|
241
|
+
) as conn:
|
|
242
|
+
with conn.cursor() as cur:
|
|
243
|
+
cur.execute(
|
|
244
|
+
"INSERT INTO access_control_policies (user_id, scopes) VALUES (%s, %s) "
|
|
245
|
+
"ON CONFLICT (user_id) DO UPDATE SET scopes = EXCLUDED.scopes",
|
|
246
|
+
(user_id, scope_list),
|
|
247
|
+
)
|
|
248
|
+
conn.commit()
|
|
249
|
+
return raw
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def seed_route_rows(resources: StackResources, rows: Sequence[tuple[str, str, str | None]]) -> None:
|
|
253
|
+
"""Upsert ``(url, scope_id, pattern)`` rows into the Postgres route store the
|
|
254
|
+
access-control verifier reads. ``pattern`` is a regex for a dynamic mapping or
|
|
255
|
+
``None`` for an exact-path mapping."""
|
|
256
|
+
with psycopg.connect(
|
|
257
|
+
host=resources.pg_host,
|
|
258
|
+
port=resources.pg_port,
|
|
259
|
+
user=resources.pg_user,
|
|
260
|
+
password=resources.pg_password,
|
|
261
|
+
dbname=resources.pg_db,
|
|
262
|
+
) as conn:
|
|
263
|
+
with conn.cursor() as cur:
|
|
264
|
+
cur.executemany(
|
|
265
|
+
"INSERT INTO access_control_routes (url, scope_id, pattern) VALUES (%s, %s, %s) "
|
|
266
|
+
"ON CONFLICT (url) DO UPDATE SET scope_id = EXCLUDED.scope_id, pattern = EXCLUDED.pattern",
|
|
267
|
+
list(rows),
|
|
268
|
+
)
|
|
269
|
+
conn.commit()
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# The two-tier route mapping the browser-e2e studio stack runs. Tier one
|
|
273
|
+
# (``ACCESS_CONTROL_PATH_PATTERNS``): a request-path regex names a route TEMPLATE. Tier two
|
|
274
|
+
# (the PG route store seeded by ``seed_studio_auth``): each template resolves to a resource
|
|
275
|
+
# id. ``studio_authed`` (every other ``/api`` route) carries a negative lookahead excluding
|
|
276
|
+
# the two public ``/api`` shapes so the registry listing stays authed.
|
|
277
|
+
STUDIO_PATH_PATTERNS: dict[str, str] = {
|
|
278
|
+
r"/api/(?!plugins/[^/]+/studio/)(?!interactions/callback(?:/|$)).*": "studio_authed",
|
|
279
|
+
r"/(?!api(?:/|$)).*": "public_spa",
|
|
280
|
+
r"/api/plugins/[^/]+/studio/.*": "public_assets",
|
|
281
|
+
r"/api/interactions/callback(?:/.*)?": "public_callback",
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
# The single protected resource id the Studio's ``*``-scope key is authorized
|
|
285
|
+
# for; the blanket authed-``/api`` pattern resolves to it.
|
|
286
|
+
STUDIO_RESOURCE_ID = "studio"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def seed_studio_auth(infra: Infra, resources: StackResources, *, api_key: str) -> str:
|
|
290
|
+
"""Seed the browser-e2e studio stack's auth: a ``*``-scope root key pinned to
|
|
291
|
+
``api_key`` (the value Playwright pastes at ``/login``), and the tier-two
|
|
292
|
+
route→resource table the ``STUDIO_PATH_PATTERNS`` templates resolve through —
|
|
293
|
+
``studio_authed`` → the ``studio`` resource, the public SPA/asset/callback
|
|
294
|
+
templates → the public marker, and the readiness probes pinned public.
|
|
295
|
+
Returns the raw root token."""
|
|
296
|
+
raw = seed_root_identity(infra, resources, user_id="studio-root", scopes=["*"], raw=api_key)
|
|
297
|
+
seed_route_rows(
|
|
298
|
+
resources,
|
|
299
|
+
[
|
|
300
|
+
("/health", "public", None),
|
|
301
|
+
("/metrics", "public", None),
|
|
302
|
+
("/ready", "public", None),
|
|
303
|
+
("studio_authed", STUDIO_RESOURCE_ID, None),
|
|
304
|
+
("public_spa", "public", None),
|
|
305
|
+
("public_assets", "public", None),
|
|
306
|
+
("public_callback", "public", None),
|
|
307
|
+
],
|
|
308
|
+
)
|
|
309
|
+
return raw
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def seed_bootstrap_key(infra: Infra, resources: StackResources) -> str:
|
|
313
|
+
"""Seed the root ``*``-scope key plus a catch-all route→scope table so every
|
|
314
|
+
route resolves to a scope the root satisfies (enforcement denies any route
|
|
315
|
+
with no scope mapping) and the ``e2e-all`` scope is mintable for keys the
|
|
316
|
+
tests provision. Returns the raw root token."""
|
|
317
|
+
raw = seed_root_identity(infra, resources, user_id="e2e-root", scopes=["*"])
|
|
318
|
+
# Enforcement denies any route with no scope mapping, so seed the route table: pin the
|
|
319
|
+
# readiness probes (/health, /metrics) public, and map every other path to "e2e-all"
|
|
320
|
+
# via a negative-lookahead pattern that never captures the two public probes (deny-wins
|
|
321
|
+
# would otherwise re-protect them).
|
|
322
|
+
seed_route_rows(
|
|
323
|
+
resources,
|
|
324
|
+
[
|
|
325
|
+
("/health", "public", None),
|
|
326
|
+
("/metrics", "public", None),
|
|
327
|
+
("e2e-all-routes", "e2e-all", r"^/(?!health$)(?!metrics$).*$"),
|
|
328
|
+
],
|
|
329
|
+
)
|
|
330
|
+
return raw
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def seed_bridge_authz(infra: Infra, resources: StackResources) -> str:
|
|
334
|
+
"""Seed the messaging-bridge stack's auth before boot: a root ``*``-scope key plus a
|
|
335
|
+
route table that pins the unauthenticated channel webhook doors public.
|
|
336
|
+
|
|
337
|
+
The channel inbound/status doors carry their OWN signature auth (Twilio HMAC, Meta
|
|
338
|
+
X-Hub-Signature-256), so they must resolve to ``public`` or the access-control guard
|
|
339
|
+
would 403 them before the plugin's signature check runs. Every other path maps to the
|
|
340
|
+
``e2e-all`` scope the root satisfies and the tests mint keys against; the readiness
|
|
341
|
+
probes stay public so boot's readiness wait is not itself denied. Returns the raw root
|
|
342
|
+
token."""
|
|
343
|
+
raw = seed_root_identity(infra, resources, user_id="bridge-root", scopes=["*"])
|
|
344
|
+
seed_route_rows(
|
|
345
|
+
resources,
|
|
346
|
+
[
|
|
347
|
+
("/health", "public", None),
|
|
348
|
+
("/metrics", "public", None),
|
|
349
|
+
("/ready", "public", None),
|
|
350
|
+
# Every channel webhook door (twilio inbound/status, whatsapp inbound):
|
|
351
|
+
# unauthenticated at the platform edge, authenticated by the provider signature.
|
|
352
|
+
("bridge-channels", "public", r"^/api/channels/.*$"),
|
|
353
|
+
# The interactions callback door a channel adapter forwards an ask_user reply
|
|
354
|
+
# to (a server-side loopback POST that carries no token).
|
|
355
|
+
("bridge-callback", "public", r"^/api/interactions/callback(?:/.*)?$"),
|
|
356
|
+
# Everything else (the authed conversation-route CRUD, the message door, key
|
|
357
|
+
# mint, schedules) → e2e-all, excluding the public shapes so deny-wins never
|
|
358
|
+
# re-protects them.
|
|
359
|
+
(
|
|
360
|
+
"bridge-protected",
|
|
361
|
+
"e2e-all",
|
|
362
|
+
r"^/(?!health$)(?!metrics$)(?!ready$)(?!api/channels/)(?!api/interactions/callback).*$",
|
|
363
|
+
),
|
|
364
|
+
],
|
|
365
|
+
)
|
|
366
|
+
return raw
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def seed_payments_authz(infra: Infra, resources: StackResources) -> str:
|
|
370
|
+
"""Seed the Stripe payments stack's auth before boot: a root ``*``-scope key plus a
|
|
371
|
+
route table that pins the two unauthenticated payment doors public.
|
|
372
|
+
|
|
373
|
+
Both doors are unauthenticated by nature — the Stripe webhook ingress carries no
|
|
374
|
+
platform API key (the topic's ``stripe`` verifier checks its signature) and the bridge
|
|
375
|
+
callback door authenticates with the question's own ``X-TAI-Bridge-Secret`` verifier —
|
|
376
|
+
so both must resolve to ``public`` or the access-control guard would 403 them before the
|
|
377
|
+
signature/header check runs. Every other path — ``/mcp`` included — maps to the
|
|
378
|
+
``e2e-all`` scope the root satisfies; the readiness probes stay public so boot's
|
|
379
|
+
readiness wait is not itself denied. Every public shape is excluded from the protected
|
|
380
|
+
catch-all so deny-wins never re-protects it. Returns the raw root token."""
|
|
381
|
+
raw = seed_root_identity(infra, resources, user_id="payments-root", scopes=["*"])
|
|
382
|
+
seed_route_rows(
|
|
383
|
+
resources,
|
|
384
|
+
[
|
|
385
|
+
("/health", "public", None),
|
|
386
|
+
("/metrics", "public", None),
|
|
387
|
+
("/ready", "public", None),
|
|
388
|
+
# The Stripe webhook ingress: unauthenticated at the platform edge, authenticated
|
|
389
|
+
# by the topic's stripe-signature verifier.
|
|
390
|
+
("payments-webhook", "public", r"^/universal_webhook/.*$"),
|
|
391
|
+
# The interaction callback door the bridge POSTs the answer to (its own
|
|
392
|
+
# X-TAI-Bridge-Secret verifier is the authentication).
|
|
393
|
+
("payments-callback", "public", r"^/api/interactions/callback(?:/.*)?$"),
|
|
394
|
+
# Everything else (the verifier bind, hook register, preset create, the MCP edge)
|
|
395
|
+
# → e2e-all, excluding the public shapes so deny-wins never re-protects them.
|
|
396
|
+
(
|
|
397
|
+
"payments-protected",
|
|
398
|
+
"e2e-all",
|
|
399
|
+
r"^/(?!health$)(?!metrics$)(?!ready$)(?!universal_webhook/)(?!api/interactions/callback).*$",
|
|
400
|
+
),
|
|
401
|
+
],
|
|
402
|
+
)
|
|
403
|
+
return raw
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def seed_projection_authz(infra: Infra, resources: StackResources) -> tuple[str, str]:
|
|
407
|
+
"""Seed the projection-authz stack's two principals + route table, before boot.
|
|
408
|
+
|
|
409
|
+
Returns ``(root_token, limited_token)``. ``root`` carries the ``*`` scope; the
|
|
410
|
+
``limited`` key carries ONLY ``mcp-access``. The route table pins ``/mcp`` to the
|
|
411
|
+
``mcp-access`` scope (so the limited key's bearer PASSES the HTTP guard and
|
|
412
|
+
reaches the MCP tool edge) while every projected operation's own synthesized
|
|
413
|
+
route (e.g. ``/api/config/reload``) maps to ``e2e-all`` — which the limited key
|
|
414
|
+
LACKS. So the limited key authenticates and dispatches, then the tool-edge authz
|
|
415
|
+
check denies the specific projected op with a ``PermissionDenied``-backed
|
|
416
|
+
``ToolError``; the root key's ``*`` satisfies both and is allowed. The readiness
|
|
417
|
+
probes stay public so boot's readiness wait is not itself denied."""
|
|
418
|
+
root = seed_root_identity(infra, resources, user_id="proj-root", scopes=["*"])
|
|
419
|
+
limited = seed_root_identity(infra, resources, user_id="proj-limited", scopes=["mcp-access"])
|
|
420
|
+
seed_route_rows(
|
|
421
|
+
resources,
|
|
422
|
+
[
|
|
423
|
+
("/health", "public", None),
|
|
424
|
+
("/metrics", "public", None),
|
|
425
|
+
("/ready", "public", None),
|
|
426
|
+
# The MCP transport endpoint itself: reachable by a key carrying
|
|
427
|
+
# ``mcp-access`` (the limited key) or ``*`` (root). The exact row wins for
|
|
428
|
+
# ``/mcp``; the protected pattern below excludes it so ``/mcp`` resolves to
|
|
429
|
+
# this scope alone (not also ``e2e-all``, which deny-wins would apply).
|
|
430
|
+
("/mcp", "mcp-access", None),
|
|
431
|
+
# Every OTHER path (a projected op's synthesized route included) → e2e-all.
|
|
432
|
+
("proj-protected", "e2e-all", r"^/(?!health$)(?!metrics$)(?!ready$)(?!mcp$).*$"),
|
|
433
|
+
],
|
|
434
|
+
)
|
|
435
|
+
return root, limited
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def seed_admin_bypass_authz(infra: Infra, resources: StackResources) -> tuple[str, str]:
|
|
439
|
+
"""Seed the admin-bypass stack's two principals + a DELIBERATELY PARTIAL route table.
|
|
440
|
+
|
|
441
|
+
Returns ``(admin_token, scoped_token)``. ``admin`` carries the condition-free ``*``
|
|
442
|
+
policy that is not an owned key — the SUPER-ADMIN discriminator (``is_admin=True``)
|
|
443
|
+
the ``ResourceGuardMiddleware`` admits on an unmapped route. ``scoped`` carries ONLY
|
|
444
|
+
``bypass-probe`` (a non-``*`` scope), so it is never admin.
|
|
445
|
+
|
|
446
|
+
Unlike the other auth seeds, this table has NO catch-all pattern: it pins the readiness
|
|
447
|
+
probes public and maps one real route (``/api/manifest``) to ``bypass-probe``, but
|
|
448
|
+
deliberately leaves ``/api/tools`` (a real authenticated GET) with no row. So
|
|
449
|
+
``/api/tools`` resolves to no resource and hits the middleware's CASE A: the admin
|
|
450
|
+
discriminator is admitted by the super-admin carve-out, while the scoped key is denied
|
|
451
|
+
``Forbidden: Route not configured``. A catch-all would hide CASE A, so it is omitted."""
|
|
452
|
+
admin = seed_root_identity(infra, resources, user_id="bypass-admin", scopes=["*"])
|
|
453
|
+
scoped = seed_root_identity(infra, resources, user_id="bypass-scoped", scopes=["bypass-probe"])
|
|
454
|
+
seed_route_rows(
|
|
455
|
+
resources,
|
|
456
|
+
[
|
|
457
|
+
("/health", "public", None),
|
|
458
|
+
("/metrics", "public", None),
|
|
459
|
+
("/ready", "public", None),
|
|
460
|
+
# The one explicitly-mapped real route the scoped key is authorized for, so a
|
|
461
|
+
# non-admin denial on the UNMAPPED /api/tools is provably CASE A (not-configured),
|
|
462
|
+
# not a blanket scope denial. A pure in-process read, needs no backend worker.
|
|
463
|
+
("/api/manifest", "bypass-probe", None),
|
|
464
|
+
],
|
|
465
|
+
)
|
|
466
|
+
return admin, scoped
|