codejury 0.1.0__py3-none-any.whl

codejury/__init__.py

@@ -0,0 +1,8 @@
+"""codejury -- a general-purpose Application Security AI audit framework.
+
+Five layers: Task / (Capability + Orchestrator + Source + Agent) / Provider / Infrastructure.
+Domain knowledge lives in YAML capability files as a first-class citizen,
+aligned with OWASP ASVS.
+"""
+
+__version__ = "0.0.0"

codejury/agents/__init__.py

@@ -0,0 +1,6 @@
+"""codejury.agents -- the audit roles (finder / challenger / judge / verifier).
+
+An agent reads an AnalysisContext and emits observations. It talks to a model
+only through a Provider, never a vendor SDK, so the role logic stays independent
+of the backend.
+"""

codejury/agents/base.py

@@ -0,0 +1,21 @@
+"""Agent ABC.
+
+An agent runs once over an AnalysisContext and returns a list of observations:
+a single run typically yields several (a verifier emits one Verdict per
+sub_capability; a finder reports several Findings).
+
+The base only declares ``run``. Concrete agents take a Provider in their own
+__init__; the orchestrator constructs and addresses them by role.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Observation
+
+
+class Agent(ABC):
+    @abstractmethod
+    def run(self, ctx: AnalysisContext) -> list[Observation]: ...

codejury/agents/debate.py

@@ -0,0 +1,188 @@
+"""Finder / Challenger / Judge agents for the debate orchestrator.
+
+Each reads the artifact, the capability hints, and the accumulated history
+(prior findings and rebuttals) from the context, calls the provider once, and
+maps its JSON reply onto observations:
+
+- Finder    -> Finding (claims) + Concession (its own retractions)
+- Challenger -> Concession (moves to dismiss findings) + Finding (ones it missed)
+- Judge     -> Finding (surviving) + Concession (dismissed)
+
+The orchestrator threads history and round_num across rounds; convergence is the
+orchestrator's job, not encoded here.
+"""
+
+from __future__ import annotations
+
+from codejury.agents.base import Agent
+from codejury.agents.parsing import one_of, str_list, to_evidence, to_float
+from codejury.domain.artifact import CodeArtifact
+from codejury.domain.capability import Capability
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Concession, Finding, Observation
+from codejury.infrastructure.json_parse import extract_json_object
+from codejury.providers.base import Message, Provider
+
+_SEVERITY = {"CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"}
+
+_FINDING_SHAPE = (
+    '{"capability": "id.sub", "title": "...", "severity": "HIGH", "cwe": "CWE-89", '
+    '"description": "...", "evidence": [{"file": "...", "line": 0, "code": "..."}], "confidence": 0.0}'
+)
+
+
+class _DebateAgent(Agent):
+    """Shared provider plumbing for the three debate roles."""
+
+    role = "agent"
+    system = ""
+
+    def __init__(self, *, provider: Provider, model: str, max_tokens: int = 2048) -> None:
+        self._provider = provider
+        self._model = model
+        self._max_tokens = max_tokens
+
+    def _ask(self, prompt: str) -> dict:
+        result = self._provider.complete(
+            system=self.system,
+            messages=[Message(role="user", content=prompt)],
+            model=self._model,
+            max_tokens=self._max_tokens,
+        )
+        return extract_json_object(result.text) or {}
+
+
+class FinderAgent(_DebateAgent):
+    role = "finder"
+    system = (
+        "You are a security finder. Identify real, exploitable vulnerabilities in the code, "
+        "being precise and avoiding false positives. Respond with a single JSON object and nothing else."
+    )
+
+    def run(self, ctx: AnalysisContext) -> list[Observation]:
+        parts = ["Review the code for security vulnerabilities.", _hints(ctx.capabilities), _code(ctx.artifact)]
+        if ctx.round_num > 1 and ctx.history:
+            parts.append(_render_history(ctx.history))
+            parts.append("Concede findings the rebuttals refute, keep the valid ones, and add any you missed.")
+        parts.append(
+            'Respond as JSON: {"findings": [' + _FINDING_SHAPE + '], '
+            '"concessions": [{"target": "finding title", "reason": "..."}]}'
+        )
+        obj = self._ask("\n\n".join(p for p in parts if p))
+        return _findings(obj.get("findings"), self.role, ctx.round_num) + _concessions(
+            obj.get("concessions"), self.role, ctx.round_num
+        )
+
+
+class ChallengerAgent(_DebateAgent):
+    role = "challenger"
+    system = (
+        "You are a skeptical security reviewer. Challenge each reported finding and try to refute it "
+        "with concrete reasoning. Respond with a single JSON object and nothing else."
+    )
+
+    def run(self, ctx: AnalysisContext) -> list[Observation]:
+        parts = [
+            "Challenge the findings below. For each one you believe is a false positive, write a rebuttal. "
+            "Add new_findings for any real issue that was missed.",
+            _code(ctx.artifact),
+            _render_history(ctx.history),
+            'Respond as JSON: {"rebuttals": [{"target": "finding title", "reason": "..."}], '
+            '"new_findings": [' + _FINDING_SHAPE + "]}",
+        ]
+        obj = self._ask("\n\n".join(p for p in parts if p))
+        return _concessions(obj.get("rebuttals"), self.role, ctx.round_num) + _findings(
+            obj.get("new_findings"), self.role, ctx.round_num
+        )
+
+
+class JudgeAgent(_DebateAgent):
+    role = "judge"
+    system = (
+        "You are an impartial security judge. Weighing the findings and rebuttals, decide which findings "
+        "survive scrutiny and which are dismissed. Respond with a single JSON object and nothing else."
+    )
+
+    def run(self, ctx: AnalysisContext) -> list[Observation]:
+        parts = [
+            "Rule on the debate below. Keep findings that withstand the rebuttals; dismiss the rest.",
+            _code(ctx.artifact),
+            _render_history(ctx.history),
+            'Respond as JSON: {"surviving": [' + _FINDING_SHAPE + '], '
+            '"dismissed": [{"target": "finding title", "reason": "..."}]}',
+        ]
+        obj = self._ask("\n\n".join(p for p in parts if p))
+        return _findings(obj.get("surviving"), self.role, ctx.round_num) + _concessions(
+            obj.get("dismissed"), self.role, ctx.round_num
+        )
+
+
+def _code(artifact: CodeArtifact) -> str:
+    return f"Code under review ({artifact.path}):\n```\n{artifact.content}\n```"
+
+
+def _hints(capabilities: list[Capability]) -> str:
+    lines = []
+    for cap in capabilities:
+        for sub_name, sub in cap.sub_capabilities.items():
+            for ap in sub.anti_patterns:
+                tag = f"{ap.cwe} {ap.severity}" if ap.cwe else ap.severity
+                lines.append(f"- {cap.id}.{sub_name} [{tag}]: {ap.description}")
+    return "Look especially for:\n" + "\n".join(lines) if lines else ""
+
+
+def _render_history(history: list[Observation]) -> str:
+    findings = [o for o in history if isinstance(o, Finding)]
+    concessions = [o for o in history if isinstance(o, Concession)]
+    blocks = []
+    if findings:
+        lines = [
+            f'- [{f.produced_by} r{f.round_num}] "{f.title}" ({f.severity}{", " + f.cwe if f.cwe else ""}): '
+            f"{f.description}"
+            for f in findings
+        ]
+        blocks.append("Findings so far:\n" + "\n".join(lines))
+    if concessions:
+        lines = [f'- [{c.produced_by} r{c.round_num}] dismiss "{c.target}": {c.reason}' for c in concessions]
+        blocks.append("Rebuttals / concessions so far:\n" + "\n".join(lines))
+    return "\n\n".join(blocks)
+
+
+def _findings(items: object, produced_by: str, round_num: int) -> list[Finding]:
+    out: list[Finding] = []
+    for f in _dicts(items):
+        title = str(f.get("title", "")).strip()
+        if not title:
+            continue
+        out.append(
+            Finding(
+                produced_by=produced_by,
+                round_num=round_num,
+                capability=str(f.get("capability", "")),
+                title=title,
+                description=str(f.get("description", "")),
+                severity=one_of(f.get("severity"), _SEVERITY, "MEDIUM"),
+                cwe=str(f.get("cwe", "")),
+                evidence=to_evidence(f.get("evidence")),
+                recommendation=str(f.get("recommendation", "")),
+                matched_anti=str_list(f.get("matched_anti")),
+                confidence=to_float(f.get("confidence"), 0.5),
+            )
+        )
+    return out
+
+
+def _concessions(items: object, produced_by: str, round_num: int) -> list[Concession]:
+    out: list[Concession] = []
+    for c in _dicts(items):
+        target = str(c.get("target", "")).strip()
+        if not target:
+            continue
+        out.append(
+            Concession(produced_by=produced_by, round_num=round_num, target=target, reason=str(c.get("reason", "")))
+        )
+    return out
+
+
+def _dicts(items: object) -> list[dict]:
+    return [x for x in items if isinstance(x, dict)] if isinstance(items, list) else []

codejury/agents/mock.py

@@ -0,0 +1,38 @@
+"""MockAgent -- a minimal Agent for the dry-run and tests.
+
+It really calls the provider (so the dry-run exercises the agent -> provider
+path), then emits one Verdict per in-scope capability, parking the model's reply
+in ``reasoning``. It does no real parsing or judgement; the Phase 3 VerifierAgent
+will.
+"""
+
+from __future__ import annotations
+
+from codejury.agents.base import Agent
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Observation, Verdict
+from codejury.providers.base import Message, Provider
+
+
+class MockAgent(Agent):
+    def __init__(self, *, provider: Provider, role: str = "verifier", model: str = "mock-model") -> None:
+        self._provider = provider
+        self._role = role
+        self._model = model
+
+    def run(self, ctx: AnalysisContext) -> list[Observation]:
+        result = self._provider.complete(
+            system=f"You are a {self._role}.",
+            messages=[Message(role="user", content=ctx.artifact.content)],
+            model=self._model,
+            max_tokens=256,
+        )
+        return [
+            Verdict(
+                capability=cap.id,
+                produced_by=self._role,
+                status="UNKNOWN",
+                reasoning=result.text,
+            )
+            for cap in ctx.capabilities
+        ]

codejury/agents/parsing.py

@@ -0,0 +1,42 @@
+"""Shared coercion from loosely-typed model JSON into domain values.
+
+Agents parse model output that may omit, mistype, or invent fields. These
+helpers coerce defensively -- they never raise on bad input, they fall back.
+"""
+
+from __future__ import annotations
+
+from codejury.domain.observation import Evidence
+
+
+def str_list(value: object) -> list[str]:
+    return [str(x) for x in value] if isinstance(value, list) else []
+
+
+def to_float(value: object, default: float) -> float:
+    try:
+        return float(value)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return default
+
+
+def one_of(value: object, allowed: set[str], default: str) -> str:
+    return value if value in allowed else default
+
+
+def to_evidence(items: object) -> list[Evidence]:
+    if not isinstance(items, list):
+        return []
+    out: list[Evidence] = []
+    for e in items:
+        if not isinstance(e, dict):
+            continue
+        line = e.get("line")
+        out.append(
+            Evidence(
+                file=str(e.get("file", "")),
+                line=line if isinstance(line, int) else None,
+                code=str(e.get("code", "")),
+            )
+        )
+    return out

codejury/agents/verifier.py

@@ -0,0 +1,106 @@
+"""VerifierAgent -- check code against a capability's correct/anti patterns.
+
+It renders the capability into a prompt, calls the provider once per capability,
+and parses the JSON reply into Verdicts. It asks for one verdict per
+sub_capability including SECURE / NOT_PRESENT, so a report can say what was
+checked and what passed -- not only what failed.
+
+Parsing is defensive: a missing or malformed reply yields no verdicts rather
+than raising, and unknown status values fall back to UNKNOWN.
+"""
+
+from __future__ import annotations
+
+from codejury.agents.base import Agent
+from codejury.agents.parsing import one_of, str_list, to_evidence, to_float
+from codejury.domain.capability import Capability
+from codejury.domain.context import AnalysisContext
+from codejury.domain.observation import Observation, Verdict
+from codejury.infrastructure.json_parse import extract_json_object
+from codejury.providers.base import Message, Provider
+
+_VALID_STATUS = {"SECURE", "VULNERABLE", "PARTIAL", "NOT_PRESENT", "UNKNOWN"}
+
+_SYSTEM = (
+    "You are a security verifier. You check code against a checklist of correct and "
+    "anti patterns and rule on each dimension, reporting what is fine as well as what "
+    "is wrong. Respond with a single JSON object and nothing else."
+)
+
+_JSON_SHAPE = (
+    '{"verdicts": [{"sub_capability": "...", '
+    '"status": "SECURE|VULNERABLE|PARTIAL|NOT_PRESENT|UNKNOWN", "reasoning": "...", '
+    '"matched_correct": ["id"], "matched_anti": ["id"], '
+    '"evidence": [{"file": "path", "line": 0, "code": "..."}], "confidence": 0.0}]}'
+)
+
+
+class VerifierAgent(Agent):
+    def __init__(self, *, provider: Provider, model: str, max_tokens: int = 2048) -> None:
+        self._provider = provider
+        self._model = model
+        self._max_tokens = max_tokens
+
+    def run(self, ctx: AnalysisContext) -> list[Observation]:
+        verdicts: list[Observation] = []
+        for cap in ctx.capabilities:
+            prompt = _build_prompt(ctx.artifact.path, ctx.artifact.content, cap)
+            result = self._provider.complete(
+                system=_SYSTEM,
+                messages=[Message(role="user", content=prompt)],
+                model=self._model,
+                max_tokens=self._max_tokens,
+            )
+            verdicts.extend(_parse_verdicts(result.text, cap))
+        return verdicts
+
+
+def _render_capability(cap: Capability) -> str:
+    lines = [f"Capability: {cap.id} ({cap.name})"]
+    for sub_name, sub in cap.sub_capabilities.items():
+        lines.append(f"\nsub_capability: {sub_name}")
+        if sub.correct_patterns:
+            lines.append("  correct patterns:")
+            lines += [f"    - {p.id}: {p.description}" for p in sub.correct_patterns]
+        if sub.anti_patterns:
+            lines.append("  anti patterns:")
+            for p in sub.anti_patterns:
+                tag = f"[{p.cwe} {p.severity}]" if p.cwe else f"[{p.severity}]"
+                lines.append(f"    - {p.id} {tag}: {p.description}")
+    return "\n".join(lines)
+
+
+def _build_prompt(path: str, content: str, cap: Capability) -> str:
+    sub_names = ", ".join(cap.sub_capabilities) or "(none)"
+    return (
+        "Check the code below against this capability.\n\n"
+        f"{_render_capability(cap)}\n\n"
+        f"Code under review ({path}):\n```\n{content}\n```\n\n"
+        f"For EVERY sub_capability ({sub_names}) output one verdict, even if SECURE "
+        "or NOT_PRESENT. Cite matched pattern ids and evidence lines.\n\n"
+        "Respond with a single JSON object exactly like:\n" + _JSON_SHAPE
+    )
+
+
+def _parse_verdicts(text: str, cap: Capability) -> list[Verdict]:
+    obj = extract_json_object(text)
+    if not obj:
+        return []
+    out: list[Verdict] = []
+    for v in obj.get("verdicts", []):
+        if not isinstance(v, dict):
+            continue
+        sub = str(v.get("sub_capability", "")).strip()
+        out.append(
+            Verdict(
+                capability=f"{cap.id}.{sub}" if sub else cap.id,
+                produced_by="verifier",
+                status=one_of(v.get("status"), _VALID_STATUS, "UNKNOWN"),
+                reasoning=str(v.get("reasoning", "")),
+                matched_correct=str_list(v.get("matched_correct")),
+                matched_anti=str_list(v.get("matched_anti")),
+                evidence=to_evidence(v.get("evidence")),
+                confidence=to_float(v.get("confidence"), 0.5),
+            )
+        )
+    return out

codejury/assembly.py

@@ -0,0 +1,76 @@
+"""Assembly -- build an orchestration from a strategy name and run it over a source.
+
+Shared by the CLI and the task layer so the "which agents + which orchestrator"
+mapping and the per-artifact run loop live in one place.
+"""
+
+from __future__ import annotations
+
+import os
+
+from codejury.agents.base import Agent
+from codejury.agents.debate import ChallengerAgent, FinderAgent, JudgeAgent
+from codejury.agents.verifier import VerifierAgent
+from codejury.domain.capability import Capability
+from codejury.domain.context import AnalysisContext
+from codejury.domain.result import AnalysisResult
+from codejury.orchestrators.base import Orchestrator
+from codejury.orchestrators.debate import DebateOrchestrator
+from codejury.orchestrators.pipeline import PipelineOrchestrator
+from codejury.orchestrators.reflexion import ReflexionOrchestrator
+from codejury.orchestrators.single import SingleOrchestrator
+from codejury.providers.anthropic import AnthropicProvider
+from codejury.providers.base import Provider
+from codejury.providers.litellm import LiteLLMProvider
+from codejury.providers.openai import OpenAIProvider
+from codejury.providers.retry import RetryProvider
+from codejury.sources.base import Source
+
+STRATEGIES = ("single", "pipeline", "debate", "reflexion")
+PROVIDERS = ("anthropic", "openai", "litellm")
+DEFAULT_MODEL = os.environ.get("CODEJURY_MODEL", "claude-sonnet-4-6")
+
+
+def make_provider(name: str, *, retries: int = 0) -> Provider:
+    if name == "openai":
+        provider: Provider = OpenAIProvider()
+    elif name == "litellm":
+        provider = LiteLLMProvider()
+    else:
+        provider = AnthropicProvider()
+    if retries > 0:
+        provider = RetryProvider(provider, max_attempts=retries + 1)
+    return provider
+
+
+def build_orchestration(
+    strategy: str, *, provider: Provider, model: str, max_tokens: int
+) -> tuple[dict[str, Agent], Orchestrator]:
+    if strategy == "debate":
+        roles = (FinderAgent, ChallengerAgent, JudgeAgent)
+        agents = {cls.role: cls(provider=provider, model=model, max_tokens=max_tokens) for cls in roles}
+        return agents, DebateOrchestrator()
+    if strategy == "reflexion":
+        agents = {
+            "actor": FinderAgent(provider=provider, model=model, max_tokens=max_tokens),
+            "critic": ChallengerAgent(provider=provider, model=model, max_tokens=max_tokens),
+        }
+        return agents, ReflexionOrchestrator()
+    verifier = {"verifier": VerifierAgent(provider=provider, model=model, max_tokens=max_tokens)}
+    if strategy == "pipeline":
+        return verifier, PipelineOrchestrator()
+    return verifier, SingleOrchestrator()
+
+
+def run_over_source(
+    source: Source,
+    capabilities: list[Capability],
+    agents: dict[str, Agent],
+    orchestrator: Orchestrator,
+) -> list[tuple[str, AnalysisResult]]:
+    """Run the orchestration over each artifact, returning (path, result) per artifact."""
+    results = []
+    for artifact in source.list_artifacts():
+        ctx = AnalysisContext(artifact=artifact, capabilities=capabilities)
+        results.append((artifact.path, orchestrator.run(agents, ctx)))
+    return results