open-interpretability 0.0.20__py3-none-any.whl

openinterp/builder/__init__.py

@@ -0,0 +1,6 @@
+"""The static site and the Parquet registry, compiled from the archived records."""
+
+from openinterp.builder.compiler import BuildReport, compile_site, load_records
+from openinterp.builder.parquet_export import export_registry, records_to_table
+
+__all__ = ["BuildReport", "compile_site", "export_registry", "load_records", "records_to_table"]

openinterp/builder/agents.py

@@ -0,0 +1,260 @@
+"""The agents page: per agent model and role, its experiments, VERIFIED rate and costs.
+
+A model takes part in an experiment in one of three roles. It is the proposer
+(``submitting_agent_model``), the reviewer (``reviewer_agent_model``), or the judge that
+scored the explanation. Self-reviewed experiments are counted apart from the rate. In
+them, the proposer's model also wrote the probes. A model that passes its own probes
+inflates the rate.
+
+Records spell one model in several ways. So pages are per canonical name
+(:func:`openinterp.core.agent_names.canonical_agent`). Each old per-spelling page
+redirects to its model's page (:func:`legacy_redirects`).
+
+The same page credits the people who ran the agents (:func:`contributors_view`). A
+person's handle is the ``contributor`` that a submission or verdict records. When the
+submission records none, it is the author of the PR that merged it
+(:mod:`openinterp.builder.users`). Each person also gets a page under ``users/``.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+from openinterp.builder.hrefs import user_href
+from openinterp.builder.significance import STRONG_EVIDENCE
+from openinterp.builder.tree import LABEL_PREFIX, Experiment
+from openinterp.core.agent_names import NOTES, canonical_agent
+from openinterp.core.costs import ROLE_ORDER, by_agent
+from openinterp.core.exceptions import RegistryError
+
+RECENT_PER_AGENT = 20  # rows in an agent page's "Most recent experiments"
+RECENT_PER_USER = 10  # rows in each list of a user's page
+
+
+def _slug(name: str) -> str:
+    return re.sub(r"[^a-z0-9._]+", "-", name.lower()).strip("-") or "unknown"
+
+
+def agent_slug(model: str) -> str:
+    """The agent page's name: ``external/deepseek-flash[1m]`` → ``deepseek-flash``."""
+    return _slug(canonical_agent(model))
+
+
+def agent_href(model: str) -> str:
+    """The agent page of ``model``, by its canonical name."""
+    return f"agents/{agent_slug(model)}.html"
+
+
+def legacy_href(model: str) -> str:
+    """Return the page that a recorded spelling had before spellings were merged.
+
+    For example, ``external/deepseek-flash[1m]`` → ``agents/external-deepseek-flash-1m.html``.
+    """
+    return f"agents/{_slug(model)}.html"
+
+
+@dataclass
+class RoleStats:
+    """One agent's experiments in one role, and how many were VERIFIED."""
+
+    role: str
+    experiments: int = 0
+    verified: int = 0
+    self_reviewed: int = 0  # proposer and reviewer only
+    self_verified: int = 0
+    costs: dict | None = None  # a :func:`openinterp.core.costs.by_agent` row
+
+    @property
+    def independent(self) -> int:
+        """Experiments that another model reviewed."""
+        return self.experiments - self.self_reviewed
+
+    @property
+    def independent_verified(self) -> int:
+        """VERIFIED experiments that another model reviewed."""
+        return self.verified - self.self_verified
+
+    @property
+    def rate(self) -> float | None:
+        """VERIFIED / experiments, over independent reviews only.
+
+        A judge scores explanations after the verdict, so its experiments' verdicts aren't
+        its rate.
+        """
+        if self.role == "judge" or not self.independent:
+            return None
+        return self.independent_verified / self.independent
+
+
+@dataclass
+class AgentView:
+    """One agent model's page: its roles, spellings and recent experiments."""
+
+    model: str  # canonical name
+    href: str
+    roles: dict[str, RoleStats] = field(default_factory=dict)
+    spellings: set[str] = field(default_factory=set)  # as the records name it
+    experiments: int = 0  # experiments it took any part in, in any role
+    recent: list[dict] = field(default_factory=list)  # newest first, any role
+
+    @property
+    def role_rows(self) -> list[RoleStats]:
+        """The agent's roles in ``ROLE_ORDER``."""
+        return [self.roles[r] for r in ROLE_ORDER if r in self.roles]
+
+    @property
+    def note(self) -> str | None:
+        """The page's note on this model (for example, a remapped name), if any."""
+        return NOTES.get(self.model)
+
+    @property
+    def other_spellings(self) -> list[str]:
+        """Recorded names other than the canonical one, for the page to list."""
+        return sorted(s for s in self.spellings if s.removeprefix("external/") != self.model)
+
+
+def _roles_of(e: Experiment) -> dict[str, str]:
+    """``role → model`` for the agents behind one (reviewed) experiment, as recorded."""
+    sub, verdict = e.record.submission, e.record.verdict
+    roles = {"proposer": sub.submitting_agent_model, "reviewer": verdict.reviewer_agent_model}
+    if verdict.explanation_scores and verdict.explanation_scores.judge_model:
+        roles["judge"] = verdict.explanation_scores.judge_model
+    return roles
+
+
+def agents_view(experiments: Sequence[Experiment]) -> list[AgentView]:
+    """One view per agent model, most experiments first."""
+    agents: dict[str, AgentView] = {}
+    involved: dict[str, list[tuple[Experiment, list[str]]]] = {}
+
+    def agent(model: str) -> AgentView:
+        name = canonical_agent(model)
+        view = agents.setdefault(name, AgentView(name, agent_href(name)))
+        view.spellings.add(model)
+        return view
+
+    for e in experiments:
+        verified = e.record.is_verified
+        own = e.record.self_reviewed
+        by_model: dict[str, list[str]] = {}
+        for role, recorded in _roles_of(e).items():
+            model = agent(recorded).model
+            stats = agents[model].roles.setdefault(role, RoleStats(role))
+            stats.experiments += 1
+            stats.verified += verified
+            if role != "judge" and own:
+                stats.self_reviewed += 1
+                stats.self_verified += verified
+            by_model.setdefault(model, []).append(role)
+        for model, roles in by_model.items():
+            involved.setdefault(model, []).append((e, roles))
+
+    for row in _costs(experiments):
+        if row["agent"] in ROLE_ORDER:
+            agent(row["model"]).roles.setdefault(row["agent"], RoleStats(row["agent"])).costs = row
+
+    for model, view in agents.items():
+        rows = involved.get(model, [])
+        view.experiments = len(rows)
+        newest = sorted(rows, key=lambda r: (r[0].record.verdict.timestamp, r[0].id), reverse=True)
+        view.recent = [_recent_row(e, roles) for e, roles in newest[:RECENT_PER_AGENT]]
+
+    _check_slugs_unique(agents)
+    return sorted(agents.values(), key=lambda a: (-a.experiments, a.model))
+
+
+def _costs(experiments: Sequence[Experiment]) -> list[dict]:
+    """:func:`openinterp.core.costs.by_agent` rows, one per role and canonical name."""
+    return by_agent(
+        [
+            (e.record.verdict.status, e.record.submission.provenance.costs)
+            for e in experiments
+            if e.record.submission.provenance
+        ],
+        canonical_agent,
+    )
+
+
+def _recent_row(e: Experiment, roles: list[str]) -> dict:
+    ident = e.record.submission.identifier
+    return {
+        "e": e,
+        "roles": roles,
+        "href": e.href,
+        "model_name": ident.model_name,
+        "unit": f"L{ident.layer}/{LABEL_PREFIX[ident.kind]}{_unit_index(ident)}",
+        "pole": ident.pole,
+    }
+
+
+def _unit_index(ident) -> int | str:
+    """What follows the kind's prefix in a unit's label: a superposition's id, else its index."""
+    return getattr(ident, "superposition_id", ident.index)
+
+
+@dataclass
+class ContributorStats:
+    """One person's part in the registry, as maker of experiments and as reviewer."""
+
+    handle: str
+    experiments: int = 0  # made: proposed, or merged when no contributor is recorded
+    verified: int = 0
+    conclusive_nulls: int = 0  # well-probed rejections: strong evidence against
+    reviews: int = 0
+    made: list[dict] = field(default_factory=list)  # their newest experiments, at most 10
+    reviewed: list[dict] = field(default_factory=list)  # the newest they reviewed, at most 10
+
+    @property
+    def href(self) -> str:
+        """Their page on the site."""
+        return user_href(self.handle)
+
+
+def contributors_view(experiments: Sequence[Experiment]) -> list[ContributorStats]:
+    """Return one row per person, most experiments and reviews first.
+
+    An experiment's maker is ``Experiment.user``. That is the proposer's recorded
+    contributor, else the author of the PR that merged it (:mod:`openinterp.builder.users`). Its
+    reviewer is the verdict's recorded contributor. Each list of rows is newest first, and
+    keeps at most :data:`RECENT_PER_USER` rows. The counts include every experiment.
+    """
+    people: dict[str, ContributorStats] = {}
+    newest = sorted(experiments, key=lambda e: (e.record.verdict.timestamp, e.id), reverse=True)
+    for e in newest:
+        verdict = e.record.verdict
+        if e.user:
+            row = people.setdefault(e.user, ContributorStats(e.user))
+            row.experiments += 1
+            if len(row.made) < RECENT_PER_USER:
+                row.made.append(_recent_row(e, []))
+            row.verified += e.record.is_verified
+            row.conclusive_nulls += (
+                e.sig.direction == "refutes" and e.sig.conclusiveness >= STRONG_EVIDENCE
+            )
+        if verdict.contributor:
+            row = people.setdefault(verdict.contributor, ContributorStats(verdict.contributor))
+            row.reviews += 1
+            if len(row.reviewed) < RECENT_PER_USER:
+                row.reviewed.append(_recent_row(e, []))
+    return sorted(people.values(), key=lambda c: (-c.experiments - c.reviews, c.handle.lower()))
+
+
+def legacy_redirects(agents: Sequence[AgentView]) -> dict[str, str]:
+    """``old page → its model's page``, for every recorded spelling whose page moved."""
+    pages = {a.href for a in agents}
+    return {
+        old: a.href
+        for a in agents
+        for old in map(legacy_href, a.spellings)
+        if old != a.href and old not in pages
+    }
+
+
+def _check_slugs_unique(agents: dict[str, AgentView]) -> None:
+    seen: dict[str, str] = {}
+    for model, view in agents.items():
+        other = seen.setdefault(view.href, model)
+        if other != model:
+            raise RegistryError(f"agent models {other!r} and {model!r} share the page {view.href}")

openinterp/builder/aligned.py

@@ -0,0 +1,41 @@
+"""The dictionary latents aligned with a neuron's poles, as its page shows them."""
+
+from __future__ import annotations
+
+from openinterp.builder.tree import feature_href
+from openinterp.crossref import neuronpedia_url
+
+
+def aligned_latents(model, neuron, alignment: dict, studied: set) -> list[dict]:
+    """Return each pole's aligned dictionary latents for a neuron page.
+
+    Each latent links to its page here (if audited) and on Neuronpedia (if hosted there).
+    """
+    if not neuron.is_neuron:
+        return []
+    poles = []
+    for pole in neuron.sides:
+        rows = alignment.get(f"{model.slug}/L{neuron.layer}/N{neuron.idx}-{pole}", [])
+        views = []
+        for row in rows:
+            sides = []
+            for name in ("writes", "reads"):
+                side = getattr(row, name)
+                if side is None:
+                    continue
+                matches = []
+                for m in side.matches:
+                    href = feature_href(model.slug, neuron.layer, row.dictionary, m.feature_idx)
+                    matches.append(
+                        {
+                            "m": m,
+                            "href": href if href in studied else None,
+                            "neuronpedia": neuronpedia_url(row, m.feature_idx),
+                            "above": m.cosine > side.chance,
+                        }
+                    )
+                sides.append({"name": name, "side": side, "matches": matches})
+            views.append({"row": row, "sides": sides})
+        if views:
+            poles.append({"pole": pole, "rows": views})
+    return poles