codi-api-agent 0.3.1__py3-none-any.whl

api_agent/schemas.py

@@ -0,0 +1,250 @@
+"""Data structures that flow through the pipeline.
+
+These mirror the architecture's named artifacts: gathered Evidence, the
+provenance-tiered Citation, the per-tool-call trace, the Validator's
+Faithfulness verdict, and the final AgentResult.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Evidence:
+    """One piece of information gathered from an API call."""
+
+    id: str  # e.g. "E1"
+    tool: str
+    args: dict
+    data_type: str  # "doc" | "structured"
+    source: str  # citable URL / reference
+    content: str  # textual content used for grounding
+    ok: bool = True
+    error: str | None = None
+    # The COMPLETE result rows (every column, PII-redacted) when the backend provides them
+    # (the SQL executor does). Powers deterministic table rendering + full-width figure
+    # grounding; `content` stays the compacted string the LLM context uses.
+    rows: list | None = None
+    rows_truncated: bool = False  # True when a LIMIT cut the row set
+    # A describe()-style statistical profile computed over EVERY row SCANNED (up to
+    # Config.profile_rows), not just the `rows` shown. This is what makes "the maximum" the
+    # real maximum rather than the top of the first page — see sql_loader.describe_rows.
+    profile: dict | None = None
+    rows_scanned: int = 0  # rows the profile covers (>= len(rows) when the display was capped)
+
+
+@dataclass
+class Citation:
+    evidence_id: str
+    source: str
+    locator: str = ""  # which tool/field the claim came from
+    provenance: str = "curated"  # curated | user-supplied (Tier-1 URLs, later)
+
+
+@dataclass
+class ToolCall:
+    tool: str
+    args: dict
+    ok: bool
+    source: str = ""
+    error: str | None = None
+    detail: str = ""  # short human note on the outcome (e.g. "161 of 217 items", "no data")
+    kind: str | None = None  # normalized failure taxonomy: auth | unreachable | server_error
+
+
+@dataclass
+class Faithfulness:
+    score: float = 0.0  # fraction of claims supported by evidence (0..1) — GROUNDING
+    supported: bool = False
+    unsupported_claims: list[str] = field(default_factory=list)
+    notes: str = ""
+    judged: bool = True
+    # The evaluator's extra dimensions beyond grounding (score). answers_question = does the answer
+    # actually + fully address the question (responsiveness/completeness); sufficient_evidence = was
+    # enough real data retrieved to answer (deterministic). Both default True (nothing wrong).
+    answers_question: bool = True
+    sufficient_evidence: bool = True
+
+
+@dataclass
+class RubricDimension:
+    """One dimension of the answer scorecard (mirrors the eval rubric D1–D5)."""
+    name: str  # correctness | completeness | groundedness | safety | communication
+    score: float = 0.0                 # 0..1
+    passed: bool = True
+    method: str = "derived"            # how it was produced: llm | deterministic | derived
+    note: str = ""
+
+
+@dataclass
+class RubricReport:
+    """Five-dimension quality scorecard produced inline on every substantive answer.
+
+    Groundedness reuses the isolated faithfulness judge (unchanged); communication adds ONE
+    narrow presentation-only judge; correctness / completeness / safety are DERIVED from signals
+    the pipeline already computes. There is no live oracle at answer time, so correctness and
+    completeness measure "consistent with the retrieved evidence and addresses the question",
+    NOT correctness against external ground truth (that is what the offline eval does). The
+    headline `overall` (0–5) is derived by a fixed ladder, never assigned by an LLM."""
+    correctness: RubricDimension
+    completeness: RubricDimension
+    groundedness: RubricDimension
+    safety: RubricDimension
+    communication: RubricDimension
+    overall: int = 0                   # derived 0..5 ladder
+    judged: bool = False               # True when at least one LLM dimension actually graded
+    # ADVISORY turns only (recommendations rather than retrieved facts). Grading a recommendation
+    # on figure-groundedness is a category error — "deploy a locum tenens provider" is not a claim
+    # about the data — so advisory answers get this extra dimension instead: does every
+    # recommendation trace back to a stated finding and carry an owner, a timeframe and an
+    # expected impact? None on every non-advisory answer, and `dims()` omits it when absent so
+    # existing consumers (UI, eval grader) see exactly the five they already handle.
+    traceability: RubricDimension | None = None
+    # WHY the headline landed where it did. The ladder takes the FIRST rule that matches, so a
+    # scorecard showing five green dimensions and one amber gives the reader no way to tell which
+    # rule fired — reported live as "all score 100% except traceability, why is this 1/5?". The
+    # offline grader has carried a `reasons` list for exactly this since it was written.
+    overall_reason: str = ""
+
+    def dims(self) -> list[RubricDimension]:
+        base = [self.correctness, self.completeness, self.groundedness,
+                self.safety, self.communication]
+        return base + ([self.traceability] if self.traceability is not None else [])
+
+
+@dataclass
+class Usage:
+    """LLM token consumption for producing one answer (summed across every call — router,
+    executor turns, synthesis, judge, retries). A cache hit is ~0. `by_model` keeps a per-model
+    prompt/completion split so cost can be priced accurately even when the generator, judge and
+    router use different models."""
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    calls: int = 0  # number of LLM round-trips
+    by_model: dict = field(default_factory=dict)  # model -> [prompt_tokens, completion_tokens]
+    # Prompt tokens the provider served from its PREFIX CACHE (OpenAI reports these as
+    # usage.prompt_tokens_details.cached_tokens; they are a SUBSET of prompt_tokens, billed at a
+    # discount). Tracked because 84% of this agent's input is a stable system+tool-spec prefix —
+    # without measuring hits there is no way to tell whether caching is working at all.
+    cached_prompt_tokens: int = 0
+    cached_by_model: dict = field(default_factory=dict)  # model -> cached prompt tokens
+    # PIPELINE STAGE -> [prompt, completion, cached, calls]. Without this the only visible number
+    # is a per-query total, which hides that a handful of full-evidence passes (synthesis,
+    # validator, evaluator) dominate spend while the executor turns — the ones that DO cache —
+    # are comparatively cheap. Compaction has to be aimed, not sprayed.
+    by_stage: dict = field(default_factory=dict)
+
+    def add(self, prompt: int | None, completion: int | None, total: int | None,
+            model: str = "", cached: int | None = None, stage: str = "") -> None:
+        p, c, ca = prompt or 0, completion or 0, cached or 0
+        self.prompt_tokens += p
+        self.completion_tokens += c
+        self.total_tokens += total if total is not None else (p + c)
+        self.cached_prompt_tokens += ca
+        self.calls += 1
+        pp, cc = self.by_model.get(model, (0, 0))
+        self.by_model[model] = (pp + p, cc + c)
+        if ca:
+            self.cached_by_model[model] = self.cached_by_model.get(model, 0) + ca
+        s = self.by_stage.setdefault(stage or "other", [0, 0, 0, 0])
+        s[0] += p
+        s[1] += c
+        s[2] += ca
+        s[3] += 1
+
+    @property
+    def cache_hit_rate(self) -> float:
+        """Fraction of prompt tokens served from cache (0.0 when nothing was cached)."""
+        return (self.cached_prompt_tokens / self.prompt_tokens) if self.prompt_tokens else 0.0
+
+
+# Providers bill a cache HIT at a fraction of the normal input rate (OpenAI: 50% off for the
+# 4o/4.1/o-series; Groq's discount differs by model). Applied to the cached SUBSET of prompt
+# tokens so reported cost matches the invoice instead of over-charging every cached prefix.
+CACHED_INPUT_DISCOUNT = 0.5
+
+
+def _cost_in(prompt: int, cached: int, in_rate: float) -> float:
+    """Input $ for one model's tokens, charging the cached subset at the discounted rate."""
+    cached = min(cached, prompt)  # cached is a SUBSET of prompt_tokens — never let it exceed
+    full = prompt - cached
+    return (full / 1_000_000) * in_rate + (cached / 1_000_000) * in_rate * CACHED_INPUT_DISCOUNT
+
+
+def usage_cost(usage: "Usage", pricing: dict) -> float | None:
+    """Estimate the USD cost of `usage` from a pricing table `{model: (in_per_1M, out_per_1M)}`.
+    Matches a model name exactly, then by its base name (after the last '/', e.g. a Groq
+    'openai/gpt-4o' → 'gpt-4o'). Returns None when NO model matched the table (unknown / free
+    provider) so callers can simply hide the cost rather than show a misleading $0."""
+    if not pricing:
+        return None
+    total, matched = 0.0, False
+    for model, (p, c) in usage.by_model.items():
+        rate = pricing.get(model) or (pricing.get(model.split("/")[-1]) if model else None)
+        if rate:
+            matched = True
+            total += (_cost_in(p, usage.cached_by_model.get(model, 0), rate[0])
+                      + (c / 1_000_000) * rate[1])
+    return round(total, 6) if matched else None
+
+
+def _rate_for(model: str, pricing: dict):
+    """The (input, output) per-1M rate for `model`, matching exactly then by base name
+    (a Groq `openai/gpt-4o` falls back to `gpt-4o`). None when the model isn't priced."""
+    if not pricing or not model:
+        return pricing.get(model) if pricing else None
+    return pricing.get(model) or pricing.get(model.split("/")[-1])
+
+
+def unpriced_models(usage: "Usage", pricing: dict) -> list[str]:
+    """Models that actually consumed tokens but have NO entry in the pricing table.
+
+    Cost is summed per model, so an unpriced model contributes $0 while the total still looks
+    authoritative — the number is quietly too low rather than obviously absent. That matters more
+    since the writing stages moved to their own model: with `SYNTHESIS_MODEL` set to something
+    unpriced, roughly 42% of input tokens would vanish from the bill with no indication. Callers
+    should surface this next to any cost they display.
+    """
+    return sorted(m for m, (p, c) in (usage.by_model or {}).items()
+                  if (p or c) and not _rate_for(m, pricing))
+
+
+def usage_cost_split(usage: "Usage", pricing: dict) -> tuple[float, float] | None:
+    """Like `usage_cost` but returns (input_usd, output_usd) separately, so a UI can show what the
+    prompt vs the completion tokens cost. Returns None when no model matched the pricing table."""
+    if not pricing:
+        return None
+    in_usd, out_usd, matched = 0.0, 0.0, False
+    for model, (p, c) in usage.by_model.items():
+        rate = pricing.get(model) or (pricing.get(model.split("/")[-1]) if model else None)
+        if rate:
+            matched = True
+            in_usd += _cost_in(p, usage.cached_by_model.get(model, 0), rate[0])
+            out_usd += (c / 1_000_000) * rate[1]
+    return (round(in_usd, 6), round(out_usd, 6)) if matched else None
+
+
+@dataclass
+class AgentResult:
+    answer: str
+    status: str  # answered | partial | abstained
+    citations: list[Citation] = field(default_factory=list)
+    evidence: list[Evidence] = field(default_factory=list)
+    trace: list[ToolCall] = field(default_factory=list)
+    faithfulness: Faithfulness | None = None
+    routed_tools: list[str] = field(default_factory=list)
+    route_method: str = ""  # "all (small catalog)" | "lexical" | "llm"
+    needs_keys: list[str] = field(default_factory=list)  # hosts needing an API key
+    error: str | None = None
+    elapsed_s: float = 0.0  # wall-clock time to produce this response
+    cached: bool = False  # True when served from the query cache (no pipeline re-run)
+    answered_by: str = ""  # which agent produced this (set by the Supervisor; blank single-agent)
+    usage: Usage | None = None  # LLM tokens spent producing this answer
+    cost_usd: float | None = None  # estimated $ cost from model pricing (None if pricing unknown)
+    rubric: "RubricReport | None" = None  # five-dimension quality scorecard (None when disabled)
+    # Which data source answered, when the question named none and others publish the same metric.
+    # Deliberately OUT-OF-BAND: the reader must be able to learn it, but naming a practice-
+    # management system inside the prose is exactly the disclosure the confidentiality rule
+    # forbids. The UI renders it in the Sources panel, which is opened deliberately.
+    source_note: str = ""

api_agent/spec_convert.py

@@ -0,0 +1,86 @@
+"""Auto-detect and convert a non-OpenAPI API description to an OpenAPI dict, so the loader can
+ingest **Postman collections, RAML 0.8, and API Blueprint** directly — the agent recognises the
+format and converts it instead of failing. Uses the same npx tools as `scripts/convert_spec.py`
+(`postman-to-openapi`, `api-spec-converter`, `apib2swagger`), so Node/npx must be on PATH for those
+formats. OpenAPI/Swagger and GraphQL need no conversion.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+from urllib.parse import urlparse
+
+# Formats this module converts to OpenAPI (GraphQL/OpenAPI are handled by their own loaders).
+CONVERTIBLE = {"postman", "raml", "apib"}
+
+
+def detect_format(text: str, source: str = "") -> str:
+    """Sniff the API-description format from its content (and filename hint). Returns one of
+    postman | raml | apib | openapi | graphql | unknown."""
+    low = source.lower().rstrip("/")
+    head = text.lstrip()[:600].lower()
+    if low.endswith((".graphql", ".gql", ".graphqls", ".sdl")):
+        return "graphql"
+    if low.endswith(".apib") or head.startswith("format:"):
+        return "apib"
+    if low.endswith(".raml") or head.startswith("#%raml"):
+        return "raml"
+    if ("schema.getpostman.com" in head or '"_postman_id"' in head
+            or low.endswith("postman_collection.json")
+            or ('"info"' in head and '"item"' in head)):  # Postman v2.x shape
+        return "postman"
+    if ('"openapi"' in head or head.startswith("openapi:")
+            or '"swagger"' in head or head.startswith("swagger:")):
+        return "openapi"
+    return "unknown"
+
+
+def _run(cmd: list[str]) -> str:
+    proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+    if proc.returncode != 0 or not proc.stdout.strip():
+        msg = (proc.stderr or proc.stdout or "").strip()
+        if "Unsupported RAML version" in msg:
+            raise RuntimeError("RAML 1.0 isn't supported by the free converter (it parses RAML 0.8). "
+                               "Convert to RAML 0.8 or OpenAPI first.")
+        raise RuntimeError(f"Spec converter failed ({cmd[2] if len(cmd) > 2 else cmd}):\n{msg[:500]}")
+    return proc.stdout
+
+
+def _parse(text: str) -> dict:
+    if text.lstrip().startswith("{"):
+        return json.loads(text)
+    import yaml
+    return yaml.safe_load(text)
+
+
+def convert_to_openapi(text: str, fmt: str) -> dict:
+    """Convert Postman / RAML / API Blueprint source TEXT to an OpenAPI (or Swagger 2.0) dict."""
+    with tempfile.TemporaryDirectory() as tmp:
+        ext = {"apib": "apib", "raml": "raml", "postman": "json"}[fmt]
+        src = Path(tmp) / f"source.{ext}"
+        src.write_text(text, encoding="utf-8")
+        if fmt == "apib":
+            return _parse(_run(["npx", "--yes", "apib2swagger", "-i", str(src)]))          # -> Swagger 2.0
+        if fmt == "raml":
+            return _parse(_run(["npx", "--yes", "api-spec-converter",
+                                "--from=raml", "--to=openapi_3", str(src)]))                # -> OpenAPI 3
+        # postman-to-openapi writes to a file
+        dst = Path(tmp) / "out.yaml"
+        _run(["npx", "--yes", "postman-to-openapi@latest", str(src), "-f", str(dst)])
+        return _parse(dst.read_text(encoding="utf-8"))
+
+
+def set_base_url(spec: dict, base_url: str) -> dict:
+    """Point the converted spec at the real server (Postman/RAML often leave a `{{baseUrl}}`
+    placeholder or no host). Works for both OpenAPI 3 (`servers`) and Swagger 2.0 (host/basePath)."""
+    base_url = base_url.strip().rstrip("/")
+    if "openapi" in spec:
+        spec["servers"] = [{"url": base_url}]
+    else:  # Swagger 2.0 (apib2swagger output)
+        p = urlparse(base_url)
+        spec["host"] = p.netloc or base_url
+        spec["schemes"] = [p.scheme or "https"]
+        spec["basePath"] = p.path or "/"
+    return spec