codejury 0.1.0__py3-none-any.whl

codejury/evaluation.py

@@ -0,0 +1,107 @@
+"""Evaluation harness -- measure detection quality against labelled golden cases.
+
+A golden case is a code snippet labelled vulnerable or not for one capability.
+``evaluate`` runs the verifier over each case and scores predictions into a
+confusion matrix with precision / recall / accuracy. The metric math is provider
+-agnostic and unit-tested; real numbers need a real provider.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from codejury.agents.verifier import VerifierAgent
+from codejury.domain.artifact import CodeArtifact
+from codejury.domain.capability import Capability
+from codejury.domain.context import AnalysisContext
+from codejury.providers.base import Provider
+
+
+@dataclass(frozen=True, kw_only=True)
+class GoldenCase:
+    name: str
+    capability: str  # capability id this case exercises
+    vulnerable: bool  # the ground-truth label
+    code: str
+
+    @classmethod
+    def from_dict(cls, name: str, data: dict[str, Any]) -> GoldenCase:
+        return cls(
+            name=name,
+            capability=data["capability"],
+            vulnerable=bool(data["vulnerable"]),
+            code=data["code"],
+        )
+
+
+@dataclass
+class Metrics:
+    tp: int = 0
+    fp: int = 0
+    tn: int = 0
+    fn: int = 0
+
+    def record(self, *, actual: bool, predicted: bool) -> None:
+        if actual and predicted:
+            self.tp += 1
+        elif actual and not predicted:
+            self.fn += 1
+        elif not actual and predicted:
+            self.fp += 1
+        else:
+            self.tn += 1
+
+    @property
+    def total(self) -> int:
+        return self.tp + self.fp + self.tn + self.fn
+
+    @property
+    def precision(self) -> float:
+        predicted_positive = self.tp + self.fp
+        return self.tp / predicted_positive if predicted_positive else 0.0
+
+    @property
+    def recall(self) -> float:
+        actual_positive = self.tp + self.fn
+        return self.tp / actual_positive if actual_positive else 0.0
+
+    @property
+    def accuracy(self) -> float:
+        return (self.tp + self.tn) / self.total if self.total else 0.0
+
+
+def load_cases(directory: str | Path) -> list[GoldenCase]:
+    cases = []
+    for path in sorted(Path(directory).glob("*.yaml")):
+        with open(path, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        cases.append(GoldenCase.from_dict(path.stem, data))
+    return cases
+
+
+def evaluate(
+    cases: list[GoldenCase],
+    capabilities: list[Capability],
+    *,
+    provider: Provider,
+    model: str,
+    max_tokens: int = 2048,
+) -> Metrics:
+    by_id = {c.id: c for c in capabilities}
+    agent = VerifierAgent(provider=provider, model=model, max_tokens=max_tokens)
+    metrics = Metrics()
+    for case in cases:
+        capability = by_id.get(case.capability)
+        if capability is None:
+            raise ValueError(f"golden case {case.name!r} references unknown capability {case.capability!r}")
+        ctx = AnalysisContext(
+            artifact=CodeArtifact(kind="file", path=case.name, content=case.code),
+            capabilities=[capability],
+        )
+        predicted = any(getattr(v, "status", None) == "VULNERABLE" for v in agent.run(ctx))
+        metrics.record(actual=case.vulnerable, predicted=predicted)
+    return metrics

codejury/infrastructure/__init__.py

@@ -0,0 +1,4 @@
+"""codejury.infrastructure -- Layer 1 cross-cutting utilities (json parsing, retry, ...).
+
+Lowest layer: depends on nothing else in codejury, used by the layers above.
+"""

codejury/infrastructure/json_parse.py

@@ -0,0 +1,57 @@
+"""Best-effort extraction of a JSON object from model output.
+
+Models often wrap JSON in prose or code fences despite instructions. This
+recovers the object with no third-party dependency: try a direct parse, then a
+fenced ```json block, then the first balanced-brace span in the text.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+# Greedy so a fenced block with nested braces is captured whole.
+_FENCE = re.compile(r"```(?:json)?\s*(\{.*\})\s*```", re.DOTALL)
+
+
+def extract_json_object(text: str) -> dict | None:
+    """Return the first JSON object found in `text`, or None if there is none."""
+    text = text.strip()
+
+    try:
+        obj = json.loads(text)
+        return obj if isinstance(obj, dict) else None
+    except json.JSONDecodeError:
+        pass
+
+    fenced = _FENCE.search(text)
+    if fenced:
+        try:
+            obj = json.loads(fenced.group(1))
+            if isinstance(obj, dict):
+                return obj
+        except json.JSONDecodeError:
+            pass
+
+    return _first_balanced_object(text)
+
+
+def _first_balanced_object(text: str) -> dict | None:
+    depth = 0
+    start = -1
+    for i, ch in enumerate(text):
+        if ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}" and depth:
+            depth -= 1
+            if depth == 0:
+                try:
+                    obj = json.loads(text[start : i + 1])
+                except json.JSONDecodeError:
+                    start = -1
+                    continue
+                if isinstance(obj, dict):
+                    return obj
+    return None

codejury/orchestrators/__init__.py

@@ -0,0 +1,6 @@
+"""codejury.orchestrators -- strategies for running agents over a context.
+
+single / debate / pipeline / reflexion. The strategy is the "any orchestration"
+axis; a task picks one. Each takes the same agents and context and returns an
+AnalysisResult, so they are interchangeable.
+"""

codejury/orchestrators/base.py

@@ -0,0 +1,19 @@
+"""Orchestrator ABC.
+
+An orchestrator decides how agents run over a context -- one pass, an
+adversarial debate, capability-by-capability, etc. Capabilities are read from
+``context.capabilities``, so they are not a separate argument.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from codejury.agents.base import Agent
+from codejury.domain.context import AnalysisContext
+from codejury.domain.result import AnalysisResult
+
+
+class Orchestrator(ABC):
+    @abstractmethod
+    def run(self, agents: dict[str, Agent], context: AnalysisContext) -> AnalysisResult: ...

codejury/orchestrators/debate.py

@@ -0,0 +1,57 @@
+"""DebateOrchestrator -- adversarial Finder -> Challenger -> Judge rounds.
+
+Each round the three agents run in turn, with the accumulated history and round
+number threaded into their context. The round's product is the Judge's ruling
+(surviving Findings + dismissed Concessions).
+
+Convergence is decided here, not by the Judge: the debate stops when the set of
+surviving finding titles is unchanged from the previous round, or when
+max_rounds is reached. The final result is the last round's ruling.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+from codejury.agents.base import Agent
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Finding, Observation
+from codejury.domain.result import AnalysisResult
+from codejury.orchestrators.base import Orchestrator
+
+_REQUIRED_ROLES = ("finder", "challenger", "judge")
+
+
+class DebateOrchestrator(Orchestrator):
+    def __init__(self, *, max_rounds: int = 3) -> None:
+        self._max_rounds = max_rounds
+
+    def run(self, agents: dict[str, Agent], context: AnalysisContext) -> AnalysisResult:
+        missing = [role for role in _REQUIRED_ROLES if role not in agents]
+        if missing:
+            return AnalysisResult(error=f"debate requires agents: {', '.join(missing)}")
+        finder, challenger, judge = (agents[role] for role in _REQUIRED_ROLES)
+
+        history: list[Observation] = []
+        ruling: list[Observation] = []
+        previous_survivors: frozenset[str] | None = None
+
+        for round_num in range(1, self._max_rounds + 1):
+            try:
+                history = history + finder.run(_round_ctx(context, history, round_num))
+                history = history + challenger.run(_round_ctx(context, history, round_num))
+                ruling = judge.run(_round_ctx(context, history, round_num))
+                history = history + ruling
+            except Exception as exc:
+                return AnalysisResult(observations=ruling, error=f"debate round {round_num} failed: {exc}")
+
+            survivors = frozenset(o.title for o in ruling if isinstance(o, Finding))
+            if survivors == previous_survivors:
+                break
+            previous_survivors = survivors
+
+        return AnalysisResult(observations=ruling)
+
+
+def _round_ctx(context: AnalysisContext, history: list[Observation], round_num: int) -> AnalysisContext:
+    return dataclasses.replace(context, history=history, round_num=round_num)

codejury/orchestrators/pipeline.py

@@ -0,0 +1,32 @@
+"""PipelineOrchestrator -- capability-by-capability full sweep.
+
+Each capability is checked in its own single-capability context, so a failure or
+bad reply on one capability does not abort the rest; errors are collected and
+reported together. This is the robust choice for auditing a whole repository
+across every dimension, where the single orchestrator would stop at the first
+agent error.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+from codejury.agents.base import Agent
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Observation
+from codejury.domain.result import AnalysisResult
+from codejury.orchestrators.base import Orchestrator
+
+
+class PipelineOrchestrator(Orchestrator):
+    def run(self, agents: dict[str, Agent], context: AnalysisContext) -> AnalysisResult:
+        observations: list[Observation] = []
+        errors: list[str] = []
+        for capability in context.capabilities:
+            cap_ctx = dataclasses.replace(context, capabilities=[capability])
+            for name, agent in agents.items():
+                try:
+                    observations.extend(agent.run(cap_ctx))
+                except Exception as exc:
+                    errors.append(f"{capability.id}/{name}: {exc}")
+        return AnalysisResult(observations=observations, error="; ".join(errors) or None)

codejury/orchestrators/reflexion.py

@@ -0,0 +1,58 @@
+"""ReflexionOrchestrator -- actor -> critic -> actor self-revision loop.
+
+A lighter cousin of debate: an actor produces findings, a critic pushes back,
+and the actor revises with the critique in its history. There is no judge; the
+result is the actor's final output. Iterates until the actor's findings are
+stable or max_iterations is reached.
+
+The actor and critic are ordinary agents (e.g. Finder as actor, Challenger as
+critic), so no reflexion-specific agent is needed.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+from codejury.agents.base import Agent
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Finding, Observation
+from codejury.domain.result import AnalysisResult
+from codejury.orchestrators.base import Orchestrator
+
+_REQUIRED_ROLES = ("actor", "critic")
+
+
+class ReflexionOrchestrator(Orchestrator):
+    def __init__(self, *, max_iterations: int = 2) -> None:
+        self._max_iterations = max_iterations
+
+    def run(self, agents: dict[str, Agent], context: AnalysisContext) -> AnalysisResult:
+        missing = [role for role in _REQUIRED_ROLES if role not in agents]
+        if missing:
+            return AnalysisResult(error=f"reflexion requires agents: {', '.join(missing)}")
+        actor, critic = agents["actor"], agents["critic"]
+
+        history: list[Observation] = []
+        actor_output: list[Observation] = []
+        previous_findings: frozenset[str] | None = None
+
+        for iteration in range(1, self._max_iterations + 1):
+            try:
+                actor_output = actor.run(_iter_ctx(context, history, iteration))
+                history = history + actor_output
+
+                findings = frozenset(o.title for o in actor_output if isinstance(o, Finding))
+                if findings == previous_findings:
+                    break
+                previous_findings = findings
+
+                if iteration < self._max_iterations:
+                    history = history + critic.run(_iter_ctx(context, history, iteration))
+            except Exception as exc:
+                return AnalysisResult(observations=actor_output, error=f"reflexion iteration {iteration} failed: {exc}")
+
+        return AnalysisResult(observations=actor_output)
+
+
+def _iter_ctx(context: AnalysisContext, history: list[Observation], iteration: int) -> AnalysisContext:
+    return dataclasses.replace(context, history=history, round_num=iteration)

codejury/orchestrators/single.py

@@ -0,0 +1,24 @@
+"""SingleOrchestrator -- the baseline: run each agent once and collect verdicts.
+
+The cheapest strategy. If an agent raises (e.g. a provider failure), the run
+stops and the partial observations are returned with the error recorded, rather
+than crashing the caller.
+"""
+
+from __future__ import annotations
+
+from codejury.agents.base import Agent
+from codejury.domain.context import AnalysisContext
+from codejury.domain.result import AnalysisResult
+from codejury.orchestrators.base import Orchestrator
+
+
+class SingleOrchestrator(Orchestrator):
+    def run(self, agents: dict[str, Agent], context: AnalysisContext) -> AnalysisResult:
+        observations = []
+        for name, agent in agents.items():
+            try:
+                observations.extend(agent.run(context))
+            except Exception as exc:
+                return AnalysisResult(observations=observations, error=f"agent {name!r} failed: {exc}")
+        return AnalysisResult(observations=observations)

codejury/providers/__init__.py

@@ -0,0 +1,5 @@
+"""codejury.providers -- model backends behind a single typed interface.
+
+Agents never call a vendor SDK directly; they call a Provider. This keeps the
+"any model" axis independent from the rest of the framework.
+"""

codejury/providers/anthropic.py

@@ -0,0 +1,68 @@
+"""AnthropicProvider -- Provider backed by the Anthropic Messages API.
+
+When ``cache`` is set, the system prompt is marked with an ephemeral
+cache_control block; the capability checklist is large and reused across
+artifacts, so caching it is the high-value target.
+
+The Anthropic client is injectable so the mapping and caching logic can be
+tested without the SDK or an API key. Constructed lazily otherwise, reading
+ANTHROPIC_API_KEY from the environment.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from codejury.providers.base import CompletionResult, Message, Provider
+
+
+class AnthropicProvider(Provider):
+    def __init__(self, *, api_key: str | None = None, client: Any | None = None) -> None:
+        self._api_key = api_key
+        self._client = client
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            try:
+                import anthropic
+            except ImportError as exc:
+                raise RuntimeError(
+                    "anthropic SDK not installed; run: pip install 'codejury[anthropic]'"
+                ) from exc
+            self._client = anthropic.Anthropic(api_key=self._api_key)
+        return self._client
+
+    def complete(
+        self,
+        *,
+        system: str,
+        messages: list[Message],
+        model: str,
+        max_tokens: int,
+        cache: bool = False,
+    ) -> CompletionResult:
+        system_param: Any = system
+        if cache and system:
+            system_param = [{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}]
+
+        response = self._get_client().messages.create(
+            model=model,
+            max_tokens=max_tokens,
+            system=system_param,
+            messages=[{"role": m.role, "content": m.content} for m in messages],
+        )
+        return CompletionResult(text=_extract_text(response))
+
+
+def _extract_text(response: Any) -> str:
+    content = getattr(response, "content", None)
+    if not isinstance(content, list):
+        return str(content or "")
+    parts: list[str] = []
+    for block in content:
+        text = getattr(block, "text", None)
+        if text is None and isinstance(block, dict):
+            text = block.get("text")
+        if text:
+            parts.append(str(text))
+    return "".join(parts)

codejury/providers/base.py

@@ -0,0 +1,42 @@
+"""Provider ABC and its typed input/output.
+
+Deliberately minimal: one synchronous, non-streaming ``complete``. Streaming and
+tool-calling are intentionally left out until a concrete need appears, so the
+interface does not over-commit early.
+
+``cache`` is a portable hint, not a guarantee: Anthropic supports prompt caching
+natively, OpenAI does not, LiteLLM depends on the backend. Each provider decides
+how to map the hint onto its own implementation.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Literal
+
+Role = Literal["user", "assistant"]
+
+
+@dataclass(frozen=True, kw_only=True)
+class Message:
+    role: Role
+    content: str
+
+
+@dataclass(frozen=True, kw_only=True)
+class CompletionResult:
+    text: str
+
+
+class Provider(ABC):
+    @abstractmethod
+    def complete(
+        self,
+        *,
+        system: str,
+        messages: list[Message],
+        model: str,
+        max_tokens: int,
+        cache: bool = False,
+    ) -> CompletionResult: ...

codejury/providers/litellm.py

@@ -0,0 +1,68 @@
+"""LiteLLMProvider -- Provider backed by LiteLLM, reaching many backends.
+
+LiteLLM speaks the OpenAI chat shape, so the system prompt is sent as the first
+message. ``cache`` is accepted but not applied here: prompt caching under LiteLLM
+is backend-specific, so it stays a no-op until a backend-aware mapping is needed.
+
+The completion callable is injectable so the mapping can be tested without the
+SDK or an API key.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from codejury.providers.base import CompletionResult, Message, Provider
+from codejury.providers.openai_format import choice_text
+
+
+class LiteLLMProvider(Provider):
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        api_base: str | None = None,
+        temperature: float = 0.2,
+        completion: Callable[..., Any] | None = None,
+    ) -> None:
+        self._api_key = api_key
+        self._api_base = api_base
+        self._temperature = temperature
+        self._completion = completion
+
+    def _completion_fn(self) -> Callable[..., Any]:
+        if self._completion is None:
+            try:
+                import litellm
+            except ImportError as exc:
+                raise RuntimeError("litellm not installed; run: pip install 'codejury[litellm]'") from exc
+            self._completion = litellm.completion
+        return self._completion
+
+    def complete(
+        self,
+        *,
+        system: str,
+        messages: list[Message],
+        model: str,
+        max_tokens: int,
+        cache: bool = False,
+    ) -> CompletionResult:
+        api_messages: list[dict] = []
+        if system:
+            api_messages.append({"role": "system", "content": system})
+        api_messages += [{"role": m.role, "content": m.content} for m in messages]
+
+        kwargs: dict[str, Any] = {
+            "model": model,
+            "messages": api_messages,
+            "max_tokens": max_tokens,
+            "temperature": self._temperature,
+        }
+        if self._api_key:
+            kwargs["api_key"] = self._api_key
+        if self._api_base:
+            kwargs["api_base"] = self._api_base
+
+        response = self._completion_fn()(**kwargs)
+        return CompletionResult(text=choice_text(response))

codejury/providers/mock.py

@@ -0,0 +1,32 @@
+"""MockProvider -- a Provider that returns canned text instead of calling a model.
+
+Used for the end-to-end dry-run and for tests, so the pipeline can run with no
+API key and deterministic output. It holds no parsing or audit logic: it returns
+whatever text it was configured with and records each call for inspection.
+"""
+
+from __future__ import annotations
+
+from codejury.providers.base import CompletionResult, Message, Provider
+
+
+class MockProvider(Provider):
+    def __init__(self, *, responses: list[str] | None = None, default: str = "") -> None:
+        # responses are returned in order, one per call; once exhausted, `default`
+        # is returned for every further call.
+        self._responses = list(responses or [])
+        self._default = default
+        self.calls: list[dict] = []
+
+    def complete(
+        self,
+        *,
+        system: str,
+        messages: list[Message],
+        model: str,
+        max_tokens: int,
+        cache: bool = False,
+    ) -> CompletionResult:
+        self.calls.append({"system": system, "messages": messages, "model": model})
+        text = self._responses.pop(0) if self._responses else self._default
+        return CompletionResult(text=text)

codejury/providers/openai.py

@@ -0,0 +1,57 @@
+"""OpenAIProvider -- Provider backed by the OpenAI Chat Completions API.
+
+The system prompt is sent as the first chat message. ``cache`` is accepted but
+not applied: OpenAI caches long prompts automatically server-side, with no
+request parameter to set.
+
+The client is injectable so the mapping can be tested without the SDK or a key.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from codejury.providers.base import CompletionResult, Message, Provider
+from codejury.providers.openai_format import choice_text
+
+
+class OpenAIProvider(Provider):
+    def __init__(self, *, api_key: str | None = None, base_url: str | None = None, client: Any | None = None) -> None:
+        self._api_key = api_key
+        self._base_url = base_url
+        self._client = client
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            try:
+                import openai
+            except ImportError as exc:
+                raise RuntimeError("openai not installed; run: pip install 'codejury[openai]'") from exc
+            kwargs: dict[str, Any] = {}
+            if self._api_key:
+                kwargs["api_key"] = self._api_key
+            if self._base_url:
+                kwargs["base_url"] = self._base_url
+            self._client = openai.OpenAI(**kwargs)
+        return self._client
+
+    def complete(
+        self,
+        *,
+        system: str,
+        messages: list[Message],
+        model: str,
+        max_tokens: int,
+        cache: bool = False,
+    ) -> CompletionResult:
+        api_messages: list[dict] = []
+        if system:
+            api_messages.append({"role": "system", "content": system})
+        api_messages += [{"role": m.role, "content": m.content} for m in messages]
+
+        response = self._get_client().chat.completions.create(
+            model=model,
+            messages=api_messages,
+            max_tokens=max_tokens,
+        )
+        return CompletionResult(text=choice_text(response))

codejury/providers/openai_format.py

@@ -0,0 +1,30 @@
+"""Text extraction for the OpenAI chat-completions response shape.
+
+Shared by OpenAIProvider and LiteLLMProvider, since LiteLLM returns the same
+``choices[0].message.content`` structure (a string, or a list of content blocks).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def choice_text(response: Any) -> str:
+    choices = getattr(response, "choices", None) or []
+    if not choices:
+        return ""
+    message = getattr(choices[0], "message", choices[0])
+    content = getattr(message, "content", None)
+    if content is None and isinstance(message, dict):
+        content = message.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = []
+        for block in content:
+            if isinstance(block, dict) and "text" in block:
+                parts.append(str(block["text"]))
+            elif isinstance(block, str):
+                parts.append(block)
+        return "".join(parts)
+    return str(content or "")