eq-layer 0.1.0__tar.gz → 0.2.0__tar.gz

{eq_layer-0.1.0 → eq_layer-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: eq-layer
-Version: 0.1.0
+Version: 0.2.0
 Summary: Inspectable intent, affect, repair, uncertainty, and factored-action control for LLM responses
 Project-URL: Homepage, https://github.com/Furox-Art/eq-layer
 Project-URL: Repository, https://github.com/Furox-Art/eq-layer
@@ -300,6 +300,39 @@ machine-readable records are
 `eval/results/coarse_disagreement_no_go.json`, and
 `eval/results/dbdc3_breakdown_no_go.json`.
 
+### Layer compute cost and decision quality
+
+Component metrics do not say what the layer costs to run. A paired A/B run of
+the same 15 seeded requests with the layer enabled and disabled, 100 passes
+per arm, measured the layer's own deterministic compute with no LLM in either
+arm:
+
+| arm | added median (ms) | added p95 (ms) | intent holdout accuracy (36) | appropriateness rubric |
+| --- | ---: | ---: | ---: | ---: |
+| learned intent adapter | 1.1612 | 1.4636 | 1.000 | 0.9917 |
+| heuristic zero-dependency | 0.1104 | 0.2265 | 0.2778 | 1.0000 |
+
+Classification and routing are timed independently rather than lumped: with
+the learned adapter, intent inference is ~92% of classification and routing
+is ~2% of the total. The two adapters are not interchangeable — both score
+1.000 on the 4 labelled seed cases, but only the learned adapter holds up on
+the 36-case holdout. Latency is machine-dependent and moved ~9% between runs
+on the same machine, so treat the learned figure as approximately
+1.2 ms median rather than an exact constant.
+
+The measured quantity is the layer's compute cost and the structural quality
+of its decision, not response quality: there is no model in the measured path
+to generate prose. End-to-end quality still needs the paired human evaluation
+in `eval/AB_PROTOCOL.md`.
+
+```bash
+python eval/ab_latency_eval.py
+```
+
+Full methodology, tables and run-to-run variation are in
+`docs/AB_LATENCY_BENCHMARK.md`. Machine-readable record:
+`eval/results/ab-latency.json`.
+
 
 ## Evidence provenance
 
@@ -456,6 +489,7 @@ eval/
   ab_prepare.py        A/B blinding
   ab_score.py          single-rater scoring
   ab_score_multi.py    multi-rater scoring
+  ab_latency_eval.py   layer-on vs layer-off latency + decision-quality A/B
   run.py
 ```
 

{eq_layer-0.1.0 → eq_layer-0.2.0}/README.md

@@ -269,6 +269,39 @@ machine-readable records are
 `eval/results/coarse_disagreement_no_go.json`, and
 `eval/results/dbdc3_breakdown_no_go.json`.
 
+### Layer compute cost and decision quality
+
+Component metrics do not say what the layer costs to run. A paired A/B run of
+the same 15 seeded requests with the layer enabled and disabled, 100 passes
+per arm, measured the layer's own deterministic compute with no LLM in either
+arm:
+
+| arm | added median (ms) | added p95 (ms) | intent holdout accuracy (36) | appropriateness rubric |
+| --- | ---: | ---: | ---: | ---: |
+| learned intent adapter | 1.1612 | 1.4636 | 1.000 | 0.9917 |
+| heuristic zero-dependency | 0.1104 | 0.2265 | 0.2778 | 1.0000 |
+
+Classification and routing are timed independently rather than lumped: with
+the learned adapter, intent inference is ~92% of classification and routing
+is ~2% of the total. The two adapters are not interchangeable — both score
+1.000 on the 4 labelled seed cases, but only the learned adapter holds up on
+the 36-case holdout. Latency is machine-dependent and moved ~9% between runs
+on the same machine, so treat the learned figure as approximately
+1.2 ms median rather than an exact constant.
+
+The measured quantity is the layer's compute cost and the structural quality
+of its decision, not response quality: there is no model in the measured path
+to generate prose. End-to-end quality still needs the paired human evaluation
+in `eval/AB_PROTOCOL.md`.
+
+```bash
+python eval/ab_latency_eval.py
+```
+
+Full methodology, tables and run-to-run variation are in
+`docs/AB_LATENCY_BENCHMARK.md`. Machine-readable record:
+`eval/results/ab-latency.json`.
+
 
 ## Evidence provenance
 
@@ -425,6 +458,7 @@ eval/
   ab_prepare.py        A/B blinding
   ab_score.py          single-rater scoring
   ab_score_multi.py    multi-rater scoring
+  ab_latency_eval.py   layer-on vs layer-off latency + decision-quality A/B
   run.py
 ```
 

{eq_layer-0.1.0 → eq_layer-0.2.0}/eq_layer/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 from .affect import AffectAdapter, HeuristicAffect, load_cases
 from .auditor import AuditContext, AuditIssue, ResponseAudit, ResponseAuditor, audit_from_generation_metadata

{eq_layer-0.1.0 → eq_layer-0.2.0}/eq_layer/ab_eval.py

@@ -31,8 +31,16 @@ def prepare_blinded(
     pairs: list[dict],
     *,
     seed: int = 0,
+    arm_names: tuple[str, str] = ("eq", "baseline"),
 ) -> tuple[list[dict], dict]:
-    """Randomize baseline/EQ sides and return ballot + separate decode key."""
+    """Randomize the two response sides and return ballot + separate decode key.
+
+    arm_names labels (eq, baseline) in the decode key so the key can describe an
+    arm honestly. The primary endpoint compares the EQ response against the
+    calibrated length-matched control rather than against the raw baseline, and
+    calling that control "baseline" in the artifact would misdescribe it.
+    """
+    eq_arm, control_arm = arm_names
     rng = random.Random(seed)
     ballot: list[dict] = []
     key: dict[str, dict[str, str]] = {}
@@ -53,12 +61,13 @@ def prepare_blinded(
         response_a = eq_response if eq_is_a else baseline
         response_b = baseline if eq_is_a else eq_response
         key[case_id] = {
-            "A": "eq" if eq_is_a else "baseline",
-            "B": "baseline" if eq_is_a else "eq",
+            "A": eq_arm if eq_is_a else control_arm,
+            "B": control_arm if eq_is_a else eq_arm,
         }
         ballot.append(
             {
                 "id": case_id,
+                "stratum": (row.get("source") or {}).get("stratum"),
                 "context": row.get("context", []),
                 "response_A": response_a,
                 "response_B": response_b,
@@ -132,14 +141,23 @@ def summarize(wins: int, losses: int, ties: int) -> PreferenceSummary:
     )
 
 
-def score_blinded(rated_ballot: list[dict], key: dict) -> dict:
+def score_blinded(
+    rated_ballot: list[dict],
+    key: dict,
+    *,
+    exclude_ids: set[str] | None = None,
+) -> dict:
     counts = {
         dimension: {"wins": 0, "losses": 0, "ties": 0}
         for dimension in DIMENSIONS
     }
+    excluded: list[str] = []
 
     for row in rated_ballot:
         case_id = str(row["id"])
+        if exclude_ids and case_id in exclude_ids:
+            excluded.append(case_id)
+            continue
         if case_id not in key:
             raise ValueError(f"Missing decode key for case: {case_id}")
 
@@ -166,6 +184,8 @@ def score_blinded(rated_ballot: list[dict], key: dict) -> dict:
     report = {
         "design": "blind-pairwise-human-evaluation",
         "unit": "case-level paired preference",
+        "excluded_ids": sorted(excluded),
+        "n_excluded": len(excluded),
         "dimensions": {},
     }
     for dimension, values in counts.items():

{eq_layer-0.1.0 → eq_layer-0.2.0}/eq_layer/cli.py

@@ -127,7 +127,7 @@ def _route(args: argparse.Namespace) -> int:
 def _doctor() -> int:
     payload = {
         "package": "eq-layer",
-        "version": "0.1.0",
+        "version": "0.2.0",
         "python": sys.version.split()[0],
         "ml_extra_available": True,
     }

{eq_layer-0.1.0 → eq_layer-0.2.0}/eq_layer/response_experiment.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import json
 import random
+import re
 import shlex
 import subprocess
 from dataclasses import dataclass, field
@@ -460,6 +461,117 @@ class FullEQPipeline:
         return steered, metadata
 
 
+# Surface-form limits expressed without any policy content. Used to build the
+# length-matched control arm: the model is asked for the same amount of text it
+# is asked for in the EQ arm, but receives no affect, intent, task or repair
+# decision. Without it, EQ-vs-baseline measures length as much as steering.
+#
+# Every profile is a positive target. An earlier set was phrased as negative
+# constraints ("no preamble, no caveats, no summary, no restatement") and was
+# measured on a 12-case Qwen2.5-0.5B pilot: the stronger the prohibition list,
+# the LONGER the reply. Mean characters ran 97 for a mild profile and 155 for
+# the strictest, and one case ballooned to 450 characters under an explicit
+# "reply in one or two sentences" instruction. A small model treats a stack of
+# prohibitions as material to elaborate on.
+#
+# The EQ instruction does not enumerate prohibitions, which is the likeliest
+# reason it stays short. The control has to be built the same way.
+#
+# Calibration selects among these on length alone. Tuning the control against
+# preference outcomes would fit the nuisance variable to the result it is meant
+# to be compared against.
+LENGTH_CONTROL_PROFILES: dict[str, str] = {
+    # Vague targets. Measured long on a small model: 23.7 and 25.7 mean words
+    # against an EQ arm at 14.8. Kept because they show the sentence count is
+    # what does the work, not the tone of the instruction.
+    "answer_only": (
+        "Reply with only the answer to what was asked. "
+        "Ask at most {question_budget} question(s)."
+    ),
+    "direct": (
+        "Reply with the direct answer, in one or two sentences. "
+        "Ask at most {question_budget} question(s) when the request is unclear."
+    ),
+    # Explicit sentence-count ladder. 12.2 mean words at one sentence, so the
+    # count is the dial and the intermediate rungs are what close the gap to
+    # the EQ arm rather than one aggressive profile overshooting it.
+    "three_sentence": (
+        "Reply in three sentences at most: the answer itself. "
+        "Ask at most {question_budget} question(s) instead when the request is unclear."
+    ),
+    "two_sentence": (
+        "Reply in two sentences at most: the answer itself. "
+        "Ask at most {question_budget} question(s) instead when the request is unclear."
+    ),
+    "one_sentence": (
+        "Reply in one sentence: the answer itself. "
+        "Ask at most {question_budget} question(s) instead when the request is unclear."
+    ),
+    "shortest_complete": (
+        "Reply with exactly what was asked for, in the fewest words that stay "
+        "complete. "
+        "Ask at most {question_budget} question(s)."
+    ),
+}
+
+DEFAULT_LENGTH_PROFILE = "answer_only"
+
+
+def build_length_matched_messages(
+    messages: list[dict],
+    realization: dict,
+    profile: str = DEFAULT_LENGTH_PROFILE,
+) -> list[dict]:
+    """Baseline arm with only the surface budget pinned, not the EQ policy."""
+    if profile not in LENGTH_CONTROL_PROFILES:
+        raise ValueError(
+            f"Unknown length profile {profile!r}; expected one of "
+            f"{sorted(LENGTH_CONTROL_PROFILES)}"
+        )
+    budget = realization.get("question_budget", 0)
+    instruction = LENGTH_CONTROL_PROFILES[profile].format(question_budget=budget)
+    return [{"role": "system", "content": instruction}, *messages]
+
+
+EQ_MATCHED_PROFILE = "eq_matched"
+
+
+def count_sentences(text: str) -> int:
+    parts = [part for part in re.split(r"[.!?]+", text) if part.strip()]
+    return len(parts)
+
+
+def build_eq_matched_messages(
+    messages: list[dict],
+    realization: dict,
+    eq_response: str,
+) -> list[dict]:
+    """Control arm whose sentence budget is read off the EQ arm's own output.
+
+    The fixed ladder bottoms out around 11 percent off on 120 cases because
+    "reply in one sentence" does not reliably land on one sentence, and the
+    ladder has no rung between one and two. Reading the target off the EQ
+    response closes that per case instead of on average.
+
+    This couples the two arms: the control is no longer independent of EQ. The
+    bias runs in the conservative direction. The control is told a sentence
+    count, never a decision, so it cannot inherit what EQ decided; but a
+    response shaped like EQ will more often come out similar in form, which
+    pushes disagreement toward ties. That would hide an EQ win rather than
+    manufacture one, which is the safer direction for a control to err in.
+
+    Selection still reads length only. Preference outcomes are not consulted.
+    """
+    target = min(max(count_sentences(eq_response), 1), 4)
+    budget = realization.get("question_budget", 0)
+    unit = "sentence" if target == 1 else "sentences"
+    instruction = (
+        f"Reply in {target} {unit} at most: the answer itself. "
+        f"Ask at most {budget} question(s) instead when the request is unclear."
+    )
+    return [{"role": "system", "content": instruction}, *messages]
+
+
 def generate_pairs(
     cases: Iterable[dict],
     *,
@@ -468,9 +580,18 @@ def generate_pairs(
     seed: int = 0,
     allow_annotations: bool = False,
     auditor_regenerations: int = 0,
+    length_matched_arm: bool = False,
+    length_profiles: tuple[str, ...] | None = None,
 ) -> list[dict]:
     if auditor_regenerations not in {0, 1}:
         raise ValueError("auditor_regenerations must be 0 or 1.")
+    profiles = tuple(length_profiles or ())
+    for name in profiles:
+        if name not in LENGTH_CONTROL_PROFILES and name != EQ_MATCHED_PROFILE:
+            raise ValueError(
+                f"Unknown length profile {name!r}; expected one of "
+                f"{sorted(LENGTH_CONTROL_PROFILES)}"
+            )
 
     rng = random.Random(seed)
     output: list[dict] = []
@@ -494,12 +615,43 @@ def generate_pairs(
             )
 
             pair_seed = stable_case_seed(seed, case_id)
-            conditions = ["baseline", "eq"]
-            rng.shuffle(conditions)
-            responses: dict[str, str] = {}
+            realization = (eq_meta.get("factored_action") or {}).get("realization") or {}
+            # One arm per profile so a single generation pass covers the whole
+            # calibration sweep. Each arm is an independent condition with the
+            # same model, temperature and per-case seed.
+            controls = [f"length_matched:{name}" for name in profiles]
+            if length_matched_arm and not profiles:
+                controls.append("length_matched")
+
+            # EQ is generated first because the eq_matched control reads its
+            # sentence budget off the EQ response. Shuffling the control arms
+            # among themselves keeps them from being generated in a fixed order.
+            responses: dict[str, str] = {
+                "baseline": model.generate(messages, seed=pair_seed, condition="baseline"),
+                "eq": model.generate(eq_messages, seed=pair_seed, condition="eq"),
+            }
+            rng.shuffle(controls)
 
-            for condition in conditions:
-                condition_messages = messages if condition == "baseline" else eq_messages
+            for condition in controls:
+                if condition == "length_matched":
+                    condition_messages = build_length_matched_messages(
+                        messages,
+                        realization,
+                        DEFAULT_LENGTH_PROFILE,
+                    )
+                elif condition == "length_matched:eq_matched":
+                    condition_messages = build_eq_matched_messages(
+                        messages,
+                        realization,
+                        responses["eq"],
+                    )
+                else:
+                    profile = condition.split(":", 1)[1]
+                    condition_messages = build_length_matched_messages(
+                        messages,
+                        realization,
+                        profile,
+                    )
                 responses[condition] = model.generate(
                     condition_messages,
                     seed=pair_seed,
@@ -545,7 +697,7 @@ def generate_pairs(
                     "model_id": model.model_id,
                     "temperature": model.temperature,
                     "pair_seed": pair_seed,
-                    "order": conditions,
+                    "order": ["baseline", "eq", *controls],
                     "same_model_both_arms": True,
                     "annotations_used": bool(annotated),
                     "auditor_regenerations_allowed": auditor_regenerations,
@@ -557,6 +709,12 @@ def generate_pairs(
             }
             if regeneration_count:
                 record["eq_initial"] = initial_eq
+            for name in profiles:
+                record[f"length_matched_{name}"] = responses[f"length_matched:{name}"]
+            if length_matched_arm and not profiles:
+                record["length_matched"] = responses["length_matched"]
+            if profiles:
+                record["length_profiles"] = list(profiles)
 
             output.append(record)
     finally:
@@ -575,9 +733,38 @@ def experiment_manifest(
     allow_annotations: bool,
     git_commit: str | None = None,
     auditor_regenerations: int = 0,
+    length_matched_arm: bool = False,
+    length_profiles: tuple[str, ...] = (),
 ) -> dict:
+    profiles = list(length_profiles)
+    # Reject unknown names here as well as at generation time. The manifest is the
+    # record other people read, so an unresolvable profile must not be written
+    # into it and left to fail later.
+    unknown = [
+        name
+        for name in profiles
+        if name not in LENGTH_CONTROL_PROFILES and name != EQ_MATCHED_PROFILE
+    ]
+    if unknown:
+        raise ValueError(
+            f"Unknown length profile(s) {unknown}; expected one of "
+            f"{sorted(LENGTH_CONTROL_PROFILES)}"
+        )
+    arms = (
+        ["baseline", "eq"]
+        + [f"length_matched:{name}" for name in profiles]
+        + (["length_matched"] if length_matched_arm and not profiles else [])
+    )
     return {
-        "design": "same-base-model paired baseline-vs-eq-layer",
+        "design": (
+            "same-base-model length-calibration sweep"
+            if profiles
+            else (
+                "same-base-model three-arm baseline-vs-length-matched-vs-eq-layer"
+                if length_matched_arm
+                else "same-base-model paired baseline-vs-eq-layer"
+            )
+        ),
         "model_id": model.model_id,
         "model_command_executable": model.command[0],
         "model_command_argv_sha256": hashlib.sha256(
@@ -599,8 +786,41 @@ def experiment_manifest(
         "subtext_model_sha256": file_sha256(subtext_model),
         "eq_layer_git_commit": git_commit,
         "pairing_rule": "same model id, same sampling temperature, same per-case seed",
+        "arms": arms,
+        "length_matched_arm": length_matched_arm,
+        "length_profiles": profiles,
+        "length_profile_selection": (
+            "Chosen on arm length only. The control must not be tuned against "
+            "preference outcomes, or the nuisance variable gets fitted to the "
+            "result it exists to be compared against."
+        ),
+        "eq_matched_arm_coupling": (
+            "When eq_matched is swept, its sentence budget is read off the EQ "
+            "response, so the control is not independent of EQ. The bias runs "
+            "toward ties, not toward an EQ win: the control receives a sentence "
+            "count and never the decision."
+        )
+        if EQ_MATCHED_PROFILE in profiles
+        else None,
         "arm_difference": (
             "EQ arm prepends only the EQ-Layer system control instruction; "
             "the original transcript is preserved identically in both arms."
         ),
+        "length_control": (
+            "The length-matched arm pins only the surface budget (verbosity, "
+            "question_budget) with no affect, intent, task or repair decision. "
+            "EQ-vs-length_matched isolates the deterministic policy contribution; "
+            "EQ-vs-baseline does not, because the EQ arm runs roughly 3x shorter "
+            "than the unconstrained baseline."
+        ),
+        "claim_boundary": (
+            "EQ-vs-baseline mixes the deterministic decision with the surface budget "
+            "it implies. Only EQ-vs-length_matched measures the deterministic layer "
+            "alone, and even that leaves a measured residual: on 120 cases the best "
+            "calibrated control sat 11.2 percent off, and deriving the control's "
+            "sentence budget from the EQ response made it worse, not better, because "
+            "EQ's brevity comes from its decision rather than from a surface budget. "
+            "The primary endpoint is therefore the routing decision, scored by "
+            "eval/route_score.py, not response preference."
+        ),
     }

{eq_layer-0.1.0 → eq_layer-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eq-layer"
-version = "0.1.0"
+version = "0.2.0"
 description = "Inspectable intent, affect, repair, uncertainty, and factored-action control for LLM responses"
 readme = "README.md"
 license = { text = "MIT" }