arcus-cli 0.1.0__py3-none-any.whl

arcus/adapters/arc_adapter.py

@@ -0,0 +1,68 @@
+import os
+from enum import Enum
+
+from openai import OpenAI
+from openai.types.chat import ChatCompletion
+
+
+class ArcModel(str, Enum):
+    """The four models ARC currently exposes."""
+
+    GPT_OSS_120B = "gpt-oss-120b"
+    GLM_5_3 = "GLM-5.3"
+    KIMI_K3 = "Kimi-K3"
+    DEEPSEEK_V4_FLASH = "DeepSeek-V4-Flash"
+
+
+class ArcAdapter:
+    """Thin wrapper around the openai SDK pointed at ARC's endpoint.
+
+    ARC's API is OpenAI-compatible, so we don't need a custom HTTP client,
+    just the right base_url and a key. Everything else (routing, caching,
+    the quality gate) should go through this instead of importing openai
+    directly, so ARC's endpoint details only live in one place.
+    """
+
+    BASE_URL = "https://llm-api.arc.vt.edu/api/v1"
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        timeout: float = 60.0,
+        max_retries: int = 2,
+    ) -> None:
+        api_key = api_key or os.environ.get("ARC_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "no ARC API key found. Pass api_key=<your-key> "
+                "Get one from llm.arc.vt.edu under "
+                "User profile > Settings > Account > API keys."
+            )
+
+        # timeout + max_retries gives us the "be a good citizen on shared
+        # infra" behavior the spec asks for, no need to hand-roll a rate
+        # limiter on top of what the SDK already does.
+        self._client = OpenAI(
+            base_url=self.BASE_URL,
+            api_key=api_key,
+            timeout=timeout,
+            max_retries=max_retries,
+        )
+
+    def chat(
+        self,
+        model: ArcModel | str,
+        messages: list[dict],
+        **kwargs,
+    ) -> ChatCompletion:
+        """Send a chat completion request to ARC for the given model.
+
+        Returns the raw ChatCompletion, not just the text, because callers
+        further down the pipeline (the quality gate especially) need
+        finish_reason and usage info, not just the message content.
+        """
+        return self._client.chat.completions.create(
+            model=model.value if isinstance(model, ArcModel) else model,
+            messages=messages,
+            **kwargs,
+        )

arcus/cache/benchmark.py

@@ -0,0 +1,159 @@
+from dataclasses import dataclass
+
+from sqlmodel import SQLModel, create_engine
+
+from arcus.cache.semantic_cache import lookup, store
+
+# true paraphrases of stable/conceptual questions, should hit the cache
+_PARAPHRASE_PAIRS = [
+    ("how does binary search work", "explain the binary search algorithm"),
+    ("what is a hash table", "can you explain what a hash map is"),
+    ("explain the difference between TCP and UDP", "what's the difference between TCP and UDP protocols"),
+    ("what is recursion in programming", "can you explain recursion"),
+    ("how does photosynthesis work", "explain the process of photosynthesis"),
+    ("what caused World War 1", "what were the causes of the First World War"),
+    ("explain the pythagorean theorem", "what is the pythagorean theorem"),
+    ("how do neural networks learn", "explain how neural networks are trained"),
+    ("what is object oriented programming", "can you explain OOP"),
+    ("how does a car engine work", "explain how an internal combustion engine works"),
+    ("what is the theory of relativity", "explain einstein's theory of relativity"),
+    ("how do vaccines work", "explain how vaccines protect against disease"),
+    ("what is dependency injection", "explain the dependency injection pattern"),
+    ("how does DNS work", "explain how domain name resolution works"),
+    ("what is a black hole", "explain what a black hole is"),
+    ("how does compound interest work", "explain compound interest"),
+]
+
+# near-duplicate phrasing, different project/assignment number, should NOT hit
+_PROJECT_NUMBER_PAIRS = [
+    (f"when is project {n} due for CS 3214", f"when is project {n + 1} due for CS 3214") for n in range(1, 9)
+]
+
+# near-duplicate phrasing, different software version, should NOT hit
+_VERSION_PAIRS = [
+    ("what's new in python 3.10", "what's new in python 3.11"),
+    ("what's new in python 3.11", "what's new in python 3.12"),
+    ("what's new in node 18", "what's new in node 20"),
+    ("what's new in java 11", "what's new in java 17"),
+    ("what's new in react 17", "what's new in react 18"),
+    ("what's new in ubuntu 22.04", "what's new in ubuntu 24.04"),
+]
+
+# near-duplicate phrasing, different year, should NOT hit
+_YEAR_PAIRS = [
+    (f"what major tech events happened in {y}", f"what major tech events happened in {y + 1}")
+    for y in range(2018, 2026)
+]
+
+# near-duplicate phrasing, different named entity, should NOT hit
+_CITY_PAIRS = [
+    ("what's the population of Paris", "what's the population of Berlin"),
+    ("what's the population of Tokyo", "what's the population of Seoul"),
+    ("what's the population of London", "what's the population of Madrid"),
+    ("what's the population of Toronto", "what's the population of Vancouver"),
+    ("what's the population of Cairo", "what's the population of Nairobi"),
+    ("what's the population of Boston", "what's the population of Chicago"),
+]
+_COMPANY_PAIRS = [
+    ("who is the CEO of Google", "who is the CEO of Microsoft"),
+    ("who is the CEO of Amazon", "who is the CEO of Apple"),
+    ("who is the CEO of Tesla", "who is the CEO of Ford"),
+    ("who is the CEO of Netflix", "who is the CEO of Disney"),
+    ("who is the CEO of Spotify", "who is the CEO of Samsung"),
+    ("who is the CEO of Sony", "who is the CEO of Nintendo"),
+]
+
+# near-duplicate phrasing, different quantity, should NOT hit
+_QUANTITY_PAIRS = [
+    (f"how many calories are in {n} slices of pizza", f"how many calories are in {n + 1} slices of pizza")
+    for n in range(1, 7)
+]
+
+# clearly unrelated topics, easy negatives
+_UNRELATED_PAIRS = [
+    ("how does binary search work", "what's a good recipe for banana bread"),
+    ("explain the pythagorean theorem", "who won the world cup in 2022"),
+    ("how do vaccines work", "what's the best way to learn guitar"),
+    ("what is a black hole", "how do I fix a flat tire"),
+    ("how does DNS work", "what's the capital of Australia"),
+    ("what is recursion in programming", "how do I make cold brew coffee"),
+]
+
+BENCHMARK_PAIRS: list[tuple[str, str, bool]] = (
+    [(a, b, True) for a, b in _PARAPHRASE_PAIRS]
+    + [
+        (a, b, False)
+        for a, b in (
+            _PROJECT_NUMBER_PAIRS
+            + _VERSION_PAIRS
+            + _YEAR_PAIRS
+            + _CITY_PAIRS
+            + _COMPANY_PAIRS
+            + _QUANTITY_PAIRS
+            + _UNRELATED_PAIRS
+        )
+    ]
+)
+
+
+@dataclass(frozen=True)
+class BenchmarkStats:
+    tp: int
+    fp: int
+    fn: int
+    tn: int
+    precision: float
+    recall: float
+
+
+def _fresh_engine():
+    engine = create_engine("sqlite://")
+    SQLModel.metadata.create_all(engine)
+    return engine
+
+
+def evaluate_pairs(
+    pairs: list[tuple[str, str, bool]],
+    use_param_diff: bool,
+    similarity_threshold: float = 0.80,
+) -> BenchmarkStats:
+    tp = fp = fn = tn = 0
+
+    for query_a, query_b, should_match in pairs:
+        # fresh engine per pair, no cross-pair contamination from earlier
+        # entries in the loop affecting a later pair's nearest match
+        engine = _fresh_engine()
+        store(query_a, response=f"answer to: {query_a}", model="gpt-oss-120b", engine=engine)
+
+        result = lookup(query_b, engine=engine, similarity_threshold=similarity_threshold, use_param_diff=use_param_diff)
+        predicted_match = result.hit
+
+        if predicted_match and should_match:
+            tp += 1
+        elif predicted_match and not should_match:
+            fp += 1
+        elif not predicted_match and should_match:
+            fn += 1
+        else:
+            tn += 1
+
+    precision = tp / (tp + fp) if (tp + fp) else 1.0
+    recall = tp / (tp + fn) if (tp + fn) else 1.0
+
+    return BenchmarkStats(tp=tp, fp=fp, fn=fn, tn=tn, precision=precision, recall=recall)
+
+
+def run_benchmark(pairs: list[tuple[str, str, bool]] = BENCHMARK_PAIRS) -> dict[str, BenchmarkStats]:
+    return {
+        "naive_cosine": evaluate_pairs(pairs, use_param_diff=False),
+        "with_param_diff": evaluate_pairs(pairs, use_param_diff=True),
+    }
+
+
+if __name__ == "__main__":
+    results = run_benchmark()
+    print(f"benchmark set: {len(BENCHMARK_PAIRS)} pairs\n")
+    for name, stats in results.items():
+        print(f"{name}:")
+        print(f"  tp={stats.tp} fp={stats.fp} fn={stats.fn} tn={stats.tn}")
+        print(f"  precision={stats.precision:.3f} recall={stats.recall:.3f}\n")

arcus/cache/semantic_cache.py

@@ -0,0 +1,168 @@
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from enum import Enum
+
+import numpy as np
+from sqlmodel import Field, Session, SQLModel, select
+
+from arcus.embeddings import embed
+from arcus.storage.db import get_engine
+
+DEFAULT_SIMILARITY_THRESHOLD = 0.80
+
+
+class Volatility(str, Enum):
+    VOLATILE = "volatile"
+    STABLE = "stable"
+
+
+_VOLATILE_MARKERS = re.compile(
+    r"\btoday\b|\bcurrent(ly)?\b|\blatest\b|\bnow\b"
+    r"|\bthis (week|month|year)\b|\brecently\b|\bright now\b",
+    re.IGNORECASE,
+)
+
+_SEVEN_DAYS_SECONDS = 7 * 24 * 60 * 60
+
+
+def classify_volatility(query: str) -> Volatility:
+    if _VOLATILE_MARKERS.search(query):
+        return Volatility.VOLATILE
+    return Volatility.STABLE
+
+
+def ttl_seconds_for(volatility: Volatility) -> int:
+    # VOLATILE gets a TTL of 0, so it expires the instant it's written
+    # and can never actually be served back. that's simpler than a
+    # separate "don't cache this" branch, the normal expiry check already
+    # handles it. 7 days for STABLE is a first guess, easy to retune once
+    # there's real usage data on how often people ask the same
+    # conceptual question again.
+    if volatility == Volatility.VOLATILE:
+        return 0
+    return _SEVEN_DAYS_SECONDS
+
+
+_NUMBER_PATTERN = re.compile(r"\b\d[\d,]*\.?\d*\b")
+# crude proper-noun/entity proxy: capitalized words not at the very start
+# of the string, where every sentence's first word would otherwise cause
+# false positives. not real NER, no NLP dependency for it, the benchmark
+# in cache/benchmark.py is what honestly measures how well this actually
+# performs rather than just asserting it does.
+_ENTITY_PATTERN = re.compile(r"(?<!^)\b[A-Z][a-zA-Z]{2,}\b")
+
+
+def extract_params(text: str) -> set[str]:
+    numbers = set(_NUMBER_PATTERN.findall(text))
+    entities = set(_ENTITY_PATTERN.findall(text.strip()))
+    return numbers | entities
+
+
+def params_conflict(query_a: str, query_b: str) -> bool:
+    return extract_params(query_a) != extract_params(query_b)
+
+
+class CacheEntry(SQLModel, table=True):
+    id: int | None = Field(default=None, primary_key=True)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    query: str
+    response: str
+    # which ARC model actually produced this response. without this, a
+    # cache hit would have no real arm to attribute stats to, "the cache"
+    # isn't one of the four models catch/reward numbers get broken down
+    # by.
+    model: str
+    ttl_seconds: int
+    # raw float32 bytes, not JSON, cheaper to store and reload than a
+    # list of floats for a 384-dim vector.
+    embedding: bytes
+
+
+def _serialize_embedding(vector: np.ndarray) -> bytes:
+    return vector.astype(np.float32).tobytes()
+
+
+def _deserialize_embedding(blob: bytes) -> np.ndarray:
+    return np.frombuffer(blob, dtype=np.float32)
+
+
+def _is_expired(entry: CacheEntry) -> bool:
+    # SQLite has no native timezone-aware datetime type, so a round-trip
+    # through it comes back naive even though it was stored as UTC.
+    # reattach the tzinfo rather than comparing naive against aware and
+    # blowing up.
+    created_at = entry.created_at
+    if created_at.tzinfo is None:
+        created_at = created_at.replace(tzinfo=UTC)
+
+    expires_at = created_at + timedelta(seconds=entry.ttl_seconds)
+    return datetime.now(UTC) > expires_at
+
+
+def store(query: str, response: str, model: str, engine=None) -> CacheEntry:
+    engine = engine or get_engine()
+
+    vector = embed([query])[0]
+    ttl_seconds = ttl_seconds_for(classify_volatility(query))
+
+    entry = CacheEntry(
+        query=query,
+        response=response,
+        model=model,
+        ttl_seconds=ttl_seconds,
+        embedding=_serialize_embedding(vector),
+    )
+
+    with Session(engine) as session:
+        session.add(entry)
+        session.commit()
+        session.refresh(entry)
+
+    return entry
+
+
+@dataclass(frozen=True)
+class CacheResult:
+    hit: bool
+    response: str | None
+    similarity: float | None
+    matched_query: str | None
+    model: str | None
+
+
+def lookup(
+    query: str,
+    engine=None,
+    similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+    use_param_diff: bool = True,
+) -> CacheResult:
+    engine = engine or get_engine()
+
+    with Session(engine) as session:
+        entries = session.exec(select(CacheEntry)).all()
+
+    live_entries = [e for e in entries if not _is_expired(e)]
+    if not live_entries:
+        return CacheResult(hit=False, response=None, similarity=None, matched_query=None, model=None)
+
+    query_embedding = embed([query])[0]
+    similarities = [
+        (float(_deserialize_embedding(e.embedding) @ query_embedding), e) for e in live_entries
+    ]
+    best_similarity, best_entry = max(similarities, key=lambda pair: pair[0])
+
+    if best_similarity < similarity_threshold:
+        return CacheResult(hit=False, response=None, similarity=best_similarity, matched_query=None, model=None)
+
+    if use_param_diff and params_conflict(query, best_entry.query):
+        return CacheResult(hit=False, response=None, similarity=best_similarity, matched_query=None, model=None)
+
+    return CacheResult(
+        hit=True,
+        response=best_entry.response,
+        similarity=best_similarity,
+        matched_query=best_entry.query,
+        model=best_entry.model,
+    )