agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/authoring_entrypoint.py

@@ -0,0 +1,189 @@
+"""Run the established ALK authoring stages for one hosted job.
+
+This is intentionally a thin process boundary over :func:`fi.alk.harness.cli._auto`.
+Contract creation, logical environment creation, and scenario generation therefore remain the
+same implementation used by the local SDK and sandbox flows.  Daytona consumes the frozen output
+afterward; this command never executes scenarios itself.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+from pathlib import Path
+import tempfile
+
+from .cli import _auto
+from .job import HarnessJob, ProviderExecutionMode, SourceKind
+from .provider_import import inspect_provider_target
+from .scenarios import load as load_written
+from .understand import PROVIDER_IMPORT_PROFILE_PATH_ENV
+
+
+def _persist_authored_scenario_count(
+    job_path: Path, job: HarnessJob, output: Path
+) -> None:
+    """Keep the frozen job in sync with adjustments applied during authoring.
+
+    The control plane may increase ``scenario_count`` while this process is already
+    running.  ``_auto`` sees that adjustment and writes the larger validated suite,
+    but the following Bundle V2 process reloads this on-disk job document.  Without
+    reconciling it here the bundler copies the original number of scenarios and the
+    platform correctly rejects preallocation because its expected count is newer.
+    """
+    authored_count = len(load_written(output))
+    if authored_count <= 0 or authored_count == job.scenario_count:
+        return
+    updated = job.model_copy(update={"scenario_count": authored_count})
+    temporary = job_path.with_suffix(f"{job_path.suffix}.tmp")
+    temporary.write_text(
+        json.dumps(updated.model_dump(mode="json"), indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    temporary.replace(job_path)
+
+
+def _load_provider_import_profile(
+    job: HarnessJob,
+    secrets_path: Path | None,
+    profile_cache_path: Path | None = None,
+) -> dict[str, object] | None:
+    inspect_connect_only_provider = (
+        job.agent.mode is ProviderExecutionMode.CONNECT_ONLY
+        and job.agent.connector.strip().lower() in {"vapi", "retell", "retell_chat"}
+    )
+    if (
+        job.agent.mode is not ProviderExecutionMode.PROVIDER_IMPORT
+        and not inspect_connect_only_provider
+    ):
+        return None
+    if profile_cache_path is not None and profile_cache_path.is_file():
+        cached = json.loads(profile_cache_path.read_text(encoding="utf-8"))
+        if not isinstance(cached, dict):
+            raise RuntimeError("provider_import_authoring_profile_invalid")
+        return cached
+    if secrets_path is None:
+        raise RuntimeError("provider_import_authoring_secrets_missing")
+    try:
+        values = json.loads(secrets_path.read_text(encoding="utf-8"))
+    finally:
+        # The provider credential is needed only for this read-only inspection. Remove the file
+        # before any model session or source/environment process starts.
+        secrets_path.unlink(missing_ok=True)
+    if not isinstance(values, dict):
+        raise RuntimeError("provider_import_authoring_secrets_invalid")
+    connector = job.agent.connector.strip().lower()
+    provider = "retell" if connector == "retell_chat" else connector
+    secret_name = "VAPI_API_KEY" if provider == "vapi" else "RETELL_API_KEY"
+    target_key = "assistant_id" if provider == "vapi" else "agent_id"
+    profile = inspect_provider_target(
+        provider,
+        source_target_id=str(job.agent.config.get(target_key) or ""),
+        api_key=str(values.get(secret_name) or ""),
+        api_base_url=str(job.agent.config.get("provider_api_base_url") or "") or None,
+        target_modality="chat" if connector == "retell_chat" else "voice",
+    )
+    if profile_cache_path is not None:
+        profile_cache_path.write_text(
+            json.dumps(profile, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
+    return profile
+
+
+def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("job", type=Path)
+    parser.add_argument("--source", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument(
+        "--adjustments",
+        type=Path,
+        help="JSONL inbox for user corrections applied at safe stage boundaries",
+    )
+    parser.add_argument(
+        "--target-secrets",
+        type=Path,
+        help="One-shot control-process secrets used to inspect an imported provider target",
+    )
+    parser.add_argument(
+        "--provider-profile-cache",
+        type=Path,
+        help="Control-owned sanitized profile reused across authoring retries",
+    )
+    args = parser.parse_args(argv)
+
+    job = HarnessJob.model_validate(json.loads(args.job.read_text(encoding="utf-8")))
+    profile = _load_provider_import_profile(
+        job, args.target_secrets, args.provider_profile_cache
+    )
+    # Transport kinds such as ``archive`` and ``github`` describe how the platform acquired the
+    # source. Once extracted, they are repositories. A source-free connect-only provider is the
+    # exception: its fetched definition is the source of truth and must never be represented by
+    # the intentionally empty /work/source directory.
+    source_free_provider = (
+        job.source.kind is SourceKind.PROVIDER
+        and job.agent.mode is ProviderExecutionMode.CONNECT_ONLY
+        and profile is not None
+    )
+    source_kind = (
+        "provider"
+        if source_free_provider
+        else str(job.metadata.get("source_kind") or "repo")
+    )
+    if source_kind not in {"repo", "spec", "provider"}:
+        source_kind = "repo"
+    namespace = argparse.Namespace(
+        path=str(args.source.resolve()),
+        name=str(job.metadata.get("agent_name") or args.source.name),
+        kind=source_kind,
+        out=str(args.output.resolve()),
+        count=job.scenario_count,
+        model=None,
+        run_model=None,
+        job=job,
+        adjustments_path=str(args.adjustments) if args.adjustments else None,
+        authoring_only=True,
+        provider_profile=profile if source_free_provider else None,
+    )
+    previous_profile_path = os.environ.get(PROVIDER_IMPORT_PROFILE_PATH_ENV)
+    with tempfile.TemporaryDirectory(prefix="alk-provider-profile-") as temporary:
+        if profile is not None:
+            profile_path = Path(temporary) / "provider-import-profile.json"
+            profile_path.write_text(
+                json.dumps(profile, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+            )
+            os.environ[PROVIDER_IMPORT_PROFILE_PATH_ENV] = str(profile_path)
+        try:
+            status = asyncio.run(_auto(namespace))
+            if status == 0 and validate_runtime:
+                from .authoring_runtime_validation import validate_and_repair
+
+                _persist_authored_scenario_count(args.job, job, args.output.resolve())
+                runtime_job = HarnessJob.model_validate_json(args.job.read_text())
+                if profile is not None:
+                    (args.output / "provider-import-profile.json").write_text(
+                        json.dumps(profile) + "\n", encoding="utf-8"
+                    )
+                asyncio.run(
+                    validate_and_repair(
+                        runtime_job, args.source.resolve(), args.output.resolve()
+                    )
+                )
+        finally:
+            if previous_profile_path is None:
+                os.environ.pop(PROVIDER_IMPORT_PROFILE_PATH_ENV, None)
+            else:
+                os.environ[PROVIDER_IMPORT_PROFILE_PATH_ENV] = previous_profile_path
+    if status == 0:
+        _persist_authored_scenario_count(args.job, job, args.output.resolve())
+        if profile is not None:
+            (args.output.resolve() / "provider-import-profile.json").write_text(
+                json.dumps(profile, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+            )
+    return status
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fi/alk/harness/authoring_runtime_validation.py

@@ -0,0 +1,267 @@
+"""Validate generated setup against the actual hosted runtime before accepting authoring.
+
+This is setup proof, not a claim that a reference tool trajectory executed. Actual
+agent tool execution remains evidence collected during the calls.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import random
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+
+class RuntimeValidationError(RuntimeError):
+    def __init__(self, phase: str, detail: str):
+        self.phase = phase
+        super().__init__(detail)
+
+
+async def validate_once(
+    job,
+    source: Path,
+    authoring: Path,
+    *,
+    secrets_path=Path("/run/futureagi/secrets.json"),
+) -> int:
+    from . import outbound
+    from .bundle_author_v2 import author_bundle_v2
+    from .hosted_entrypoint import (
+        ProcessWorldFactory,
+        _resolve_hosted_public_url,
+        job_secret_purposes,
+    )
+    from .hosted_scheduler import _classify_ready, _run_phase
+    from .job import ProviderExecutionMode, SourceKind
+    from .process_preflight import preflight_bundle
+    from .process_runtime import ProcessRuntimeProvider
+    from .scenario_source import load_scenarios
+    from .source_data_invariants import author_invariants, check_invariants
+
+    external_provider = (
+        getattr(getattr(job, "source", None), "kind", None) is SourceKind.PROVIDER
+        and getattr(getattr(job, "agent", None), "mode", None)
+        is ProviderExecutionMode.CONNECT_ONLY
+    )
+
+    # The real execution consumes its credential file. Validation gets a private copy,
+    # with the same purpose map, so it cannot destroy the execution handoff.
+    with tempfile.TemporaryDirectory(
+        prefix="runtime-validation-", dir=authoring.parent
+    ) as root:
+        work = Path(root)
+        secrets = work / "secrets.json"
+        secrets.write_bytes(secrets_path.read_bytes())
+        secrets.chmod(0o600)
+        secret_values = tuple(
+            str(value) for value in json.loads(secrets.read_text()).values()
+        )
+        capabilities = outbound.load_capabilities(unlink=False)
+        transport = outbound.RequestsTransport()
+        provider = ProcessRuntimeProvider(
+            secrets_path=secrets,
+            secret_purpose_map=job_secret_purposes(job),
+            user_resolver=lambda _name: None,
+            require_declared_user=False,
+            public_url_resolver=lambda port, ttl: _resolve_hosted_public_url(
+                capabilities, transport, port=port, expires_in_seconds=ttl
+            ),
+            provider_attempt_id=capabilities.attempt_id,
+            provider_expires_at=capabilities.expires_at,
+        )
+        executor = ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="runtime-validation"
+        )
+        phase = "environment"
+        try:
+            bundle = work / "bundle"
+            manifest = await asyncio.to_thread(
+                author_bundle_v2,
+                source=source,
+                job=job,
+                authoring=authoring,
+                output=bundle,
+            )
+            preflight_bundle(
+                bundle, manifest, parallelism=1, secret_refs=job_secret_purposes(job)
+            )
+            runtimes = await provider.provision(
+                manifest,
+                source=source,
+                bundle_dir=bundle,
+                work_directory=work,
+                instances=1,
+            )
+            factory = ProcessWorldFactory(work)
+            runtime = runtimes[0]
+            phase = "scenarios"
+            scenarios = await asyncio.to_thread(load_scenarios, bundle)
+            if len(scenarios) != job.scenario_count:
+                raise RuntimeValidationError(
+                    phase, "Runtime scenario count differs from the requested count"
+                )
+
+            async def check_setups(invariants):
+                failures = []
+                for scenario in scenarios:
+                    try:
+                        await check_setup(scenario, invariants)
+                    except RuntimeValidationError as exc:
+                        failures.append(str(exc))
+                if failures:
+                    raise RuntimeValidationError(phase, "\n".join(failures))
+
+            async def check_setup(scenario, invariants):
+                await provider.reset(runtime, work_directory=work)
+                world = await factory.create(runtime, rng=random.Random(job.seed or 0))
+                for name, fn, target, timeout in (
+                    ("setup", scenario.setup, world, 30.0),
+                    ("ready", scenario.ready, world.read_only(), 15.0),
+                ):
+                    result = await _run_phase(
+                        fn, target, timeout=timeout, phase=name, executor=executor
+                    )
+                    if result.failure:
+                        raise RuntimeValidationError(
+                            phase, f"{scenario.scenario_key}: {name}: {result.failure}"
+                        )
+                    if name == "setup":
+                        try:
+                            await check_invariants(
+                                world.read_only(),
+                                invariants,
+                                scenario_key=scenario.scenario_key,
+                            )
+                        except Exception as exc:
+                            raise RuntimeValidationError(
+                                phase, f"{scenario.scenario_key}: source data: {exc}"
+                            ) from exc
+                    if name == "ready":
+                        verdict = _classify_ready(result.value)
+                        if verdict.broken or not verdict.held:
+                            raise RuntimeValidationError(
+                                phase,
+                                f"{scenario.scenario_key}: ready precondition did not hold",
+                            )
+
+            # Collect all executable setup errors before spending a model review or
+            # a repair attempt. Each scenario still gets an independent clean world.
+            await check_setups([])
+            if external_provider:
+                # A connect-only provider owns its state and executes its tools outside
+                # this sandbox. There is no harness-owned source database to probe or
+                # seed, so source-data invariant review would invent a local environment.
+                print(
+                    "runtime validation: external provider black-box mode; "
+                    "skipping local source-data invariant review",
+                    flush=True,
+                )
+                return len(scenarios)
+            phase = "environment"
+            await provider.reset(runtime, work_directory=work)
+            baseline = await factory.create(runtime, rng=random.Random(job.seed or 0))
+            print("runtime validation: reviewing source data invariants", flush=True)
+            invariants = await author_invariants(
+                source, authoring, baseline.read_only(), endpoints=runtime.endpoints
+            )
+            # Review probes may have effects; none belongs in the test baseline.
+            await provider.reset(runtime, work_directory=work)
+            baseline = await factory.create(runtime, rng=random.Random(job.seed or 0))
+            await check_invariants(baseline.read_only(), invariants)
+            phase = "scenarios"
+            if invariants:
+                await check_setups(invariants)
+            return len(scenarios)
+        except RuntimeValidationError as exc:
+            raise RuntimeValidationError(
+                exc.phase,
+                outbound.redact_outbound_text(
+                    str(exc), extra_secret_values=secret_values
+                ),
+            ) from None
+        except Exception as exc:
+            if "CERTIFICATE_VERIFY_FAILED" in str(exc):
+                # Generated data cannot repair the infrastructure trust store.
+                phase = "infrastructure"
+            raise RuntimeValidationError(
+                phase,
+                outbound.redact_outbound_text(
+                    f"{type(exc).__name__}: {exc}", extra_secret_values=secret_values
+                ),
+            ) from None
+        finally:
+            executor.shutdown(wait=False, cancel_futures=True)
+            await provider.close(work_directory=work)
+
+
+async def validate_and_repair(
+    job, source: Path, authoring: Path, *, validate=validate_once, repair=None
+) -> None:
+    """Two repairs per phase; environment repairs cannot exhaust setup's budget."""
+    if repair is None:
+
+        async def repair(phase, guidance):
+            from .cli import _build, _scenarios
+
+            if phase == "environment":
+                return await _build(
+                    argparse.Namespace(
+                        name=source.name,
+                        path=str(source),
+                        out=str(authoring),
+                        interactive=False,
+                        guidance=[guidance],
+                        skip_source_provision=True,
+                        external_runtime=(
+                            job.source.kind.value == "provider"
+                            and getattr(job.agent.mode, "value", None) == "connect_only"
+                        ),
+                    )
+                )
+            return await _scenarios(
+                argparse.Namespace(
+                    name=source.name,
+                    out=str(authoring),
+                    count=job.scenario_count,
+                    interactive=False,
+                    guidance=[guidance],
+                )
+            )
+
+    repairs = {"environment": 0, "scenarios": 0}
+    for attempt in range(5):
+        print(f"runtime validation: attempt {attempt + 1}/5", flush=True)
+        try:
+            count = await validate(job, source, authoring)
+        except RuntimeValidationError as exc:
+            print(f"runtime validation: {exc.phase}: {exc}", flush=True)
+            if exc.phase not in repairs or repairs[exc.phase] >= 2:
+                raise
+            repairs[exc.phase] += 1
+            guidance = (
+                "Actual hosted runtime validation failed. Repair the generated environment/data "
+                "or scenario setup using the submitted source as authority. Do not modify source, "
+                "disable database constraints, weaken checks or ready conditions, drop scenarios, "
+                "or replace tools with invented implementations. Preserve scenario count and intent. "
+                f"Diagnostic: {str(exc)[:4000]}"
+            )
+            if await repair(exc.phase, guidance):
+                raise
+        else:
+            (authoring / "runtime-validation.json").write_text(
+                json.dumps(
+                    {
+                        "status": "passed",
+                        "attempts": attempt + 1,
+                        "setup_ready_scenarios": count,
+                        "reference_tools_proven": False,
+                    },
+                    indent=2,
+                )
+                + "\n"
+            )
+            return

fi/alk/harness/backends/README.md

@@ -0,0 +1,43 @@
+# Harness backends
+
+A stage of this harness is a conversation: a system prompt, tools, a turn budget, and a loop
+that feeds tool results back until the model stops. A backend is whoever runs that loop.
+
+```
+ALK_HARNESS=claude          # default; Claude Code loop, exactly the pre-seam behaviour
+ALK_HARNESS=vertex-gemini   # Google's ADK against Vertex (location=global)
+ALK_HARNESS_MODEL=...       # optional; unset means the backend's own default
+ALK_VERTEX_LOCATION=...     # vertex-gemini only; defaults to global (Gemini 3.x lives there)
+```
+
+## The seam
+
+- `base.py` is the whole contract. `SessionSpec` is what a stage asks for; the reply
+  vocabulary (`SessionOpened`, `ModelReply`, `ToolReturned`, `StageDone`) is what a session
+  emits; `HarnessBackend` / `HarnessSession` are the two protocols a backend implements.
+- Tools are declared once, neutrally (`tool`, `tool_server`), and each backend adapts them:
+  the Claude backend builds an in-process MCP server, the Gemini backend builds function
+  declarations. Gating follows the same split: hook-based on Claude, structural on Gemini
+  (an ungranted tool is never declared).
+- `Stage` (in `session.py`) drives any backend and renders the replies into events. It never
+  names a vendor.
+
+## Adding a backend
+
+Write a module with a class exposing `name`, `default_model`, `can_drive(model)` and
+`create(spec) -> HarnessSession`, where the session yields the neutral replies and ends every
+exchange with a `StageDone`. Register it in `__init__.py` (or call `register` from anywhere).
+Nothing else in the harness changes: every stage, gate, and artifact works as-is. That is the
+slot a Bedrock, Azure, or Gemini-CLI backend drops into.
+
+Backends load lazily, so one backend's SDK is never imported because a different one ran.
+
+## What the Gemini backend supplies itself
+
+The ADK owns the loop, tool execution, and session history; the backend only adapts a
+``ToolSpec`` through ADK's ``BaseTool`` extension point and translates its event stream into
+the neutral replies. Claude Code ships Read/Glob/Grep and an operator-question tool; ADK has
+no coding-CLI file tools, so `files.py` implements the read-only file tools once for any
+backend that needs them. AskUserQuestion is deliberately not declared on the Gemini backend
+yet: unattended runs never call it, and declaring a tool the backend cannot answer would cost
+the model a turn finding that out.

fi/alk/harness/backends/__init__.py

@@ -0,0 +1,122 @@
+"""Harness backends, selected by name.
+
+``ALK_HARNESS`` picks the backend the way ``ALK_HARNESS_MODEL`` already picks the model. With
+nothing set the choice is ``vertex-gemini``, so a machine holding only Google credentials runs
+without being told to; ``ALK_HARNESS=claude`` selects the Claude Code loop instead.
+
+Backends load lazily: choosing one never imports the other's SDK, so a deployment installs only
+the provider it uses. A new backend is a module implementing ``HarnessBackend`` plus one
+``register`` call, from anywhere; nothing else in the harness changes.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Callable
+
+from .base import (
+    ASK_TOOL,
+    FILE_TOOLS,
+    KNOWN_BUILTINS,
+    Call,
+    HarnessBackend,
+    HarnessSession,
+    ModelReply,
+    Say,
+    SessionOpened,
+    SessionSpec,
+    StageDone,
+    ToolReturned,
+    ToolServer,
+    ToolSpec,
+    qualified,
+    tool,
+    tool_server,
+)
+
+__all__ = [
+    "ASK_TOOL",
+    "FILE_TOOLS",
+    "KNOWN_BUILTINS",
+    "Call",
+    "HarnessBackend",
+    "HarnessSession",
+    "ModelReply",
+    "Say",
+    "SessionOpened",
+    "SessionSpec",
+    "StageDone",
+    "ToolReturned",
+    "ToolServer",
+    "ToolSpec",
+    "qualified",
+    "tool",
+    "tool_server",
+    "register",
+    "resolve",
+    "backend_names",
+]
+
+DEFAULT_BACKEND = "vertex-gemini"
+
+_LOADERS: dict[str, Callable[[], HarnessBackend]] = {}
+_ALIASES = {
+    "gemini": "vertex-gemini",
+    "vertex_gemini": "vertex-gemini",
+    "vertexai-gemini": "vertex-gemini",
+    "claude-code": "claude",
+}
+_LIVE: dict[str, HarnessBackend] = {}
+
+
+def register(name: str, loader: Callable[[], HarnessBackend]) -> None:
+    """Make a backend selectable by name. Loader runs on first use, not at registration."""
+    _LOADERS[name] = loader
+
+
+def _load_claude() -> HarnessBackend:
+    from .claude import ClaudeBackend
+
+    return ClaudeBackend()
+
+
+def _load_vertex_gemini() -> HarnessBackend:
+    from .vertex_gemini import VertexGeminiBackend
+
+    return VertexGeminiBackend()
+
+
+register("claude", _load_claude)
+register("vertex-gemini", _load_vertex_gemini)
+
+
+def backend_names() -> list[str]:
+    return sorted(_LOADERS)
+
+
+def resolve(name: str | None = None) -> HarnessBackend:
+    """The backend a run will use: the one named, or ALK_HARNESS, or the default.
+
+    An unknown name is a loud error naming what exists. Falling back silently would run a whole
+    suite on the wrong harness, which is only discovered from the bill.
+    """
+    asked = (name or os.environ.get("ALK_HARNESS") or DEFAULT_BACKEND).strip().lower()
+    asked = _ALIASES.get(asked, asked)
+    if asked not in _LOADERS:
+        raise ValueError(
+            f"no harness backend named {asked!r}; installed backends: "
+            f"{', '.join(backend_names())}"
+        )
+    if asked not in _LIVE:
+        _LIVE[asked] = _LOADERS[asked]()
+    backend = _LIVE[asked]
+    # A named model that this backend cannot reach is a configuration mistake, and it is only
+    # visible here. Left to run, the provider rejects the model mid-stage and the failure reads
+    # as the harness having nothing to say rather than as the wrong pairing.
+    wanted = os.environ.get("ALK_HARNESS_MODEL", "").strip()
+    if wanted and not backend.can_drive(wanted):
+        raise ValueError(
+            f"harness backend {backend.name!r} cannot drive model {wanted!r}; "
+            f"its default is {backend.default_model!r}"
+        )
+    return backend