attocode-code-intel 0.1.23__tar.gz → 0.1.24__tar.gz

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: attocode-code-intel
-Version: 0.1.23
+Version: 0.1.24
 Summary: Local code search and navigation for coding agents and people
 Project-URL: Homepage, https://github.com/eren23/attocode
 Project-URL: Repository, https://github.com/eren23/attocode
@@ -156,6 +156,8 @@ To use the latest source, install from a clone of this repository: `uv tool inst
 
 ## What your agent gets
 
+Version 0.1.24 adds `ATTOCODE_INTEL_RANKING_PROVIDER=openrouter`, a preset for the opt-in hosted rerank. In a frozen trial with 24 short questions, the rerank found 24 of 26 labeled files in the first five results, against 20 without it; see [Hosted rerank](https://github.com/eren23/attocode/blob/main/docs/intelligence-search.md#hosted-rerank-experimental). Rerank stays off by default, because it sends source excerpts to OpenRouter.
+
 Version 0.1.23 is listed in the official MCP Registry and documents a `uvx` quick start. The opt-in hosted rerank now accepts the dated model ID that OpenRouter returns; see [Hosted rerank](https://github.com/eren23/attocode/blob/main/docs/intelligence-search.md#hosted-rerank-experimental). Rerank stays off by default.
 
 Version 0.1.22 adds an opt-in discovery budget of up to 100,000 files

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/README.md

@@ -43,6 +43,8 @@ To use the latest source, install from a clone of this repository: `uv tool inst
 
 ## What your agent gets
 
+Version 0.1.24 adds `ATTOCODE_INTEL_RANKING_PROVIDER=openrouter`, a preset for the opt-in hosted rerank. In a frozen trial with 24 short questions, the rerank found 24 of 26 labeled files in the first five results, against 20 without it; see [Hosted rerank](https://github.com/eren23/attocode/blob/main/docs/intelligence-search.md#hosted-rerank-experimental). Rerank stays off by default, because it sends source excerpts to OpenRouter.
+
 Version 0.1.23 is listed in the official MCP Registry and documents a `uvx` quick start. The opt-in hosted rerank now accepts the dated model ID that OpenRouter returns; see [Hosted rerank](https://github.com/eren23/attocode/blob/main/docs/intelligence-search.md#hosted-rerank-experimental). Rerank stays off by default.
 
 Version 0.1.22 adds an opt-in discovery budget of up to 100,000 files

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "attocode-code-intel"
-version = "0.1.23"
+version = "0.1.24"
 description = "Local code search and navigation for coding agents and people"
 readme = "README.md"
 requires-python = ">=3.12"

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/src/attocode_intel/__init__.py

@@ -49,4 +49,4 @@ def __getattr__(name: str) -> Any:
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 # The other copy is packages/code-intel/pyproject.toml. A test checks that both match.
-__version__ = "0.1.23"
+__version__ = "0.1.24"

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/src/attocode_intel/service.py

@@ -60,6 +60,14 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+# Settings of the frozen hosted-rerank trial (evals/results/2026-11-01-hosted-rerank.md).
+_OPENROUTER_PRESET = {
+    "endpoint": "https://openrouter.ai/api/alpha/decisions",
+    "model": "typesafe/jev-1.13",
+    "auth_env": "OPENROUTER_API_KEY",
+    "timeout_ms": 1500,
+}
+
 # Registry of service instances by project_dir
 _instances: dict[str, CodeIntelService] = {}
 _instances_lock = threading.Lock()
@@ -91,6 +99,11 @@ class CodeIntelService:
         self._ranking_provider = (self._config.ranking_provider
                                   if config is not None else
                                   os.environ.get("ATTOCODE_INTEL_RANKING_PROVIDER", "")) or "auto"
+        # The "openrouter" preset is SystemOne with the trial-tested defaults. Explicit
+        # settings still win, and the remote opt-in stays scoped to one workspace.
+        preset = _OPENROUTER_PRESET if self._ranking_provider == "openrouter" else {}
+        if preset:
+            self._ranking_provider = "systemone"
         self._systemone_status = "not_configured"
         self._memory_store: MemoryStore | None = None
         self._temporal_analyzer: TemporalCouplingAnalyzer | None = None
@@ -119,11 +132,15 @@ class CodeIntelService:
                 remote_workspace = (self._config.ranking_remote_workspace or
                                     environment.get("ATTOCODE_INTEL_RANKING_REMOTE_WORKSPACE", ""))
                 timeout_ms = (self._config.ranking_timeout_ms if config is not None else
-                              int(environment.get("ATTOCODE_INTEL_RANKING_TIMEOUT_MS", "0")))
+                              int(environment.get("ATTOCODE_INTEL_RANKING_TIMEOUT_MS", "0"))
+                              ) or preset.get("timeout_ms", 0)
                 self._systemone_ranker = SystemOneChoiceReranker(
-                    self._config.ranking_endpoint or environment.get("ATTOCODE_INTEL_RANKING_ENDPOINT", ""),
-                    model=self._config.ranking_model or environment.get("ATTOCODE_INTEL_RANKING_MODEL", ""),
-                    auth_env=self._config.ranking_auth_env or environment.get("ATTOCODE_INTEL_RANKING_AUTH_ENV", ""),
+                    self._config.ranking_endpoint or environment.get("ATTOCODE_INTEL_RANKING_ENDPOINT", "")
+                    or preset.get("endpoint", ""),
+                    model=self._config.ranking_model or environment.get("ATTOCODE_INTEL_RANKING_MODEL", "")
+                    or preset.get("model", ""),
+                    auth_env=self._config.ranking_auth_env or environment.get("ATTOCODE_INTEL_RANKING_AUTH_ENV", "")
+                    or preset.get("auth_env", ""),
                     allow_remote=(remote_opt_in and os.path.isabs(remote_workspace) and
                                   os.path.realpath(remote_workspace) == self._project_dir),
                     service_mode=self._config.is_service_mode,

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/tests/test_mcp_search_evaluation.py

@@ -1,10 +1,14 @@
 """The quality operator must measure the consumer's actual budgeted output."""
+import argparse
+import hashlib
+import json
 from copy import deepcopy
 
 import pytest
 from attocode_intel.gateway import OperationGateway
 
-from eval.quality.mcp_search import compare, evidence_hits, observe
+from eval.quality.mcp_search import compare, evidence_hits, observe, run, worst_of
+from eval.quality.rerank_trial import decide
 
 
 def test_visible_evidence_requires_literal_quote_in_the_same_file_and_snippet_lines():
@@ -379,3 +383,207 @@ def test_test_support_development_rejection_replays_from_delivery():
     assert decision["decision"] == "rejected on development: no judged test recovered at five"
     assert decision["fresh_validation_started"] is False
     assert not replayed["individual_losses"]
+
+
+def test_worst_of_keeps_only_labels_every_sample_delivered():
+    samples = [{"hits": {"files": [0, 1], "visible": [1]}}, {"hits": {"files": [1, 2], "visible": [1]}},
+               {"hits": {"files": [1], "visible": []}}]
+    assert worst_of(samples) == {"files": [1], "visible": []}
+
+
+RERANK = {"provider": "systemone", "model": "typesafe/jev-1.13", "remote": True,
+          "method": "systemone_choice", "rerank_elapsed_ms": 600}
+SCOPED = {**RERANK, "method": "source_aware_deterministic", "fallback_reason": "single_candidate"}
+del SCOPED["rerank_elapsed_ms"]
+
+
+def hosted_runs():
+    """A passing trial: +3 first-five files across two packs, no losses, all hosted."""
+    questions = (("p", "one", [0, 1]), ("q", "two", [0]))
+    design = json.dumps({"repositories": [{"pack": pack, "commit": "c", "queries": [{"id": qid}]}
+                                          for pack, qid, _ in questions]}).encode()
+    arms = {}
+    for arm in ("baseline", "candidate"):
+        arms[arm] = []
+        for pack, qid, gained in questions:
+            row = {"id": qid, "pack": pack, "query": qid, "evidence": [{"file": "a"}] * 3,
+                   "inspection_hits": [], "pages": {}}
+            for page in ("5", "24", "scoped"):
+                files = (gained if arm == "candidate" else []) if page == "5" else [0, 1, 2]
+                hits = {"files": files, "ranges": files, "visible": []}
+                row["pages"][page] = {"request": {"top_k": page}, "seconds": .5, "hits": hits}
+                if arm == "candidate":
+                    row["pages"][page]["samples"] = [
+                        {"seconds": .5, "hits": deepcopy(hits),
+                         "ranking_diagnostic": [dict(SCOPED if page == "scoped" else RERANK)]}
+                        for _ in range(3)]
+            arms[arm].append({"pack": pack, "arm": arm, "commit": "c", "queries": [row],
+                              "questions_sha256": hashlib.sha256(design).hexdigest(),
+                              "operator_sha256": "o", "engine_sha256": "e",
+                              "samples_may_differ": arm == "candidate",
+                              "rerank_calls": {"posted": 6 if arm == "candidate" else 0, "refused": 0}})
+    return design, arms["baseline"], arms["candidate"], {"usage": 1.0}, {"usage": 1.01}
+
+
+def candidate_pages(candidate, pages=("5", "24")):
+    return [r["queries"][0]["pages"][p] for r in candidate for p in pages]
+
+
+def set_files(page, files):
+    page["hits"]["files"] = files
+    for s in page.get("samples", []):
+        s["hits"]["files"] = files
+
+
+def test_hosted_trial_passes_and_reports_spend():
+    decision = decide(*hosted_runs())
+    assert decision["passed"], decision["checks"]
+    assert decision["hosted"]["files_5_net_by_pack"] == {"p": 2, "q": 1}
+    assert decision["hosted"]["attempts"] == 12 and decision["hosted"]["rankings"] == 18
+    assert decision["hosted"]["estimated_usd"] == .009
+
+
+def _small_gain(b, c, before, after):
+    set_files(c[0]["queries"][0]["pages"]["5"], [0])
+
+
+def _single_pack_gain(b, c, before, after):
+    set_files(c[0]["queries"][0]["pages"]["5"], [0, 1, 2])
+    set_files(c[1]["queries"][0]["pages"]["5"], [])
+
+
+def _loss_24(b, c, before, after):
+    set_files(c[0]["queries"][0]["pages"]["24"], [0, 1])
+
+
+def _loss_5(b, c, before, after):
+    set_files(b[0]["queries"][0]["pages"]["5"], [2])
+    set_files(b[1]["queries"][0]["pages"]["5"], [1])
+
+
+def _fallback(b, c, before, after):
+    for s in c[0]["queries"][0]["pages"]["5"]["samples"]:
+        s["ranking_diagnostic"] = [{**RERANK, "method": "source_aware_deterministic",
+                                    "fallback_reason": "transport_error"}]
+
+
+def _config(b, c, before, after):
+    c[0]["queries"][0]["pages"]["scoped"]["samples"][0]["ranking_diagnostic"] = [
+        {"provider": "systemone", "method": "source_aware_deterministic", "fallback_reason": "missing_credential"}]
+
+
+def _model(b, c, before, after):
+    candidate_pages(c)[0]["samples"][0]["ranking_diagnostic"][0]["model"] = "other/model"
+
+
+def _local(b, c, before, after):
+    candidate_pages(c)[0]["samples"][0]["ranking_diagnostic"][0]["remote"] = False
+
+
+def _slow_rerank(b, c, before, after):
+    for page in candidate_pages(c):
+        for s in page["samples"]:
+            s["ranking_diagnostic"][0]["rerank_elapsed_ms"] = 2000
+
+
+def _slow_search(b, c, before, after):
+    for page in candidate_pages(c):
+        for s in page["samples"]:
+            s["seconds"] = 3
+
+
+def _baseline_remote(b, c, before, after):
+    b[0]["rerank_calls"]["posted"] = 1
+
+
+def _refused(b, c, before, after):
+    c[0]["rerank_calls"]["refused"] = 1
+
+
+def _overspent(b, c, before, after):
+    after["usage"] = 6.5
+
+
+def _usage_reset(b, c, before, after):
+    after["usage"] = .5
+
+
+def _estimate(b, c, before, after):
+    c[0]["rerank_calls"]["posted"] = 7000
+
+
+@pytest.mark.parametrize("mutate, check", [
+    (_small_gain, "files_5_gain"), (_single_pack_gain, "files_5_gain"), (_loss_24, "no_file_loss_24"),
+    (_loss_5, "at_most_one_file_loss_5"), (_fallback, "fallback_rate"), (_config, "no_config_fallback"),
+    (_model, "hosted_ranker_used"), (_local, "hosted_ranker_used"), (_slow_rerank, "rerank_p95"),
+    (_slow_search, "search_p95"), (_baseline_remote, "baseline_no_remote_calls"),
+    (_refused, "no_refused_calls"), (_overspent, "spend"), (_usage_reset, "spend"), (_estimate, "spend")])
+def test_hosted_trial_gate_rejects(mutate, check):
+    design, b, c, before, after = hosted_runs()
+    mutate(b, c, before, after)
+    decision = decide(design, b, c, before, after)
+    assert not decision["checks"][check] and not decision["passed"]
+
+
+def _digest(b, c):
+    c[0]["questions_sha256"] = "x"
+
+
+def _engine(b, c):
+    c[0]["engine_sha256"] = "other"
+
+
+def _missing_pack(b, c):
+    del c[1]
+
+
+def _stable_samples(b, c):
+    c[0]["samples_may_differ"] = False
+
+
+def _two_samples(b, c):
+    candidate_pages(c)[0]["samples"].pop()
+
+
+def _lucky_draw(b, c):
+    candidate_pages(c)[0]["samples"][0]["hits"]["files"] = []
+
+
+@pytest.mark.parametrize("mutate", [_digest, _engine, _missing_pack, _stable_samples, _two_samples, _lucky_draw])
+def test_hosted_trial_rejects_mismatched_observations(mutate):
+    design, b, c, before, after = hosted_runs()
+    mutate(b, c)
+    with pytest.raises(ValueError):
+        decide(design, b, c, before, after)
+
+
+@pytest.mark.asyncio
+async def test_call_cap_refuses_before_any_rerank_request_leaves(tmp_path, monkeypatch):
+    from attocode_intel._internal.integrations.context.systemone_ranker import (
+        SystemOneChoiceReranker,
+    )
+
+    source = tmp_path / "repo"
+    source.mkdir()
+    (source / "settings.py").write_text(
+        'def load_settings(path):\n    """Load settings from a configuration file."""\n    return open(path).read()\n')
+    (source / "config.py").write_text(
+        'def read_configuration_file(path):\n    """Read settings from the configuration file."""\n    return path\n')
+    design = tmp_path / "design.json"
+    design.write_text(json.dumps({"budget": 2000, "repositories": [{"pack": "tiny", "commit": "c", "queries": [
+        {"id": "tiny:1", "query": "load settings", "evidence": [
+            {"file": "settings.py", "line": 1, "quote": "def load_settings(path):",
+             "source_sha256": hashlib.sha256((source / "settings.py").read_bytes()).hexdigest()}]}]}]}))
+    for name, value in (("PROVIDER", "systemone"), ("ENDPOINT", "http://127.0.0.1:9/decisions"),
+                        ("MODEL", "typesafe/jev-1.13")):
+        monkeypatch.setenv(f"ATTOCODE_INTEL_RANKING_{name}", value)
+    monkeypatch.delenv("ATTOCODE_INTEL_RANKING_AUTH_ENV", raising=False)
+    monkeypatch.delenv("ATTOCODE_LOCAL_ONLY", raising=False)
+    sent = []
+    monkeypatch.setattr(SystemOneChoiceReranker, "_post", lambda self, body, headers: sent.append(body))
+    args = argparse.Namespace(questions=design, source=source, output=tmp_path / "out.json", pack="tiny",
+                              arm="candidate", capture_generation=False, samples_may_differ=True,
+                              max_rerank_calls=0, require_rerank=True, warmup_timeout=30)
+    with pytest.raises(RuntimeError, match="Rerank call limit"):
+        await run(args)
+    assert sent == [] and not args.output.exists()

{attocode_code_intel-0.1.23 → attocode_code_intel-0.1.24}/tests/test_systemone_ranker.py

@@ -213,6 +213,34 @@ def test_remote_opt_in_is_scoped_to_one_workspace(tmp_path, monkeypatch):
     assert CodeIntelService(str(other))._systemone_ranker is None
 
 
+
+def test_openrouter_preset_uses_the_trial_settings_and_keeps_the_workspace_scope(tmp_path, monkeypatch):
+    monkeypatch.delenv("ATTOCODE_LOCAL_ONLY", raising=False)
+    allowed = tmp_path / "allowed"
+    other = tmp_path / "other"
+    allowed.mkdir()
+    other.mkdir()
+    for name in ("ENDPOINT", "MODEL", "AUTH_ENV", "TIMEOUT_MS"):
+        monkeypatch.delenv(f"ATTOCODE_INTEL_RANKING_{name}", raising=False)
+    monkeypatch.setenv("ATTOCODE_INTEL_RANKING_PROVIDER", "openrouter")
+    monkeypatch.setenv("ATTOCODE_INTEL_RANKING_ALLOW_REMOTE", "1")
+    monkeypatch.setenv("ATTOCODE_INTEL_RANKING_REMOTE_WORKSPACE", str(allowed))
+    service = CodeIntelService(str(allowed))
+    ranker = service._systemone_ranker
+    assert service._ranking_provider == "systemone" and service._systemone_status == "configured"
+    assert (ranker.endpoint, ranker.model, ranker.auth_env, ranker.timeout_seconds, ranker.remote) == (
+        "https://openrouter.ai/api/alpha/decisions", "typesafe/jev-1.13", "OPENROUTER_API_KEY", 1.5, True)
+    # The preset does not widen the remote opt-in.
+    assert CodeIntelService(str(other))._systemone_ranker is None
+    monkeypatch.delenv("ATTOCODE_INTEL_RANKING_ALLOW_REMOTE")
+    assert CodeIntelService(str(allowed))._systemone_status == "invalid_config"
+
+    explicit = CodeIntelService(str(allowed), CodeIntelConfig(
+        ranking_provider="openrouter", ranking_model="typesafe/other", ranking_timeout_ms=900,
+        ranking_allow_remote=True, ranking_remote_workspace=str(allowed)))
+    assert (explicit._systemone_ranker.model, explicit._systemone_ranker.timeout_seconds) == ("typesafe/other", .9)
+    assert explicit._systemone_ranker.endpoint == "https://openrouter.ai/api/alpha/decisions"
+
 def test_redirect_does_not_send_source_to_a_second_endpoint():
     requested = []