bpsai-pair 2.35.2__tar.gz → 2.37.0__tar.gz

{bpsai_pair-2.35.2 → bpsai_pair-2.37.0}/MANIFEST.in

@@ -7,3 +7,5 @@ recursive-include bpsai_pair/wizard/prompts *.md
 include bpsai_pair/licensing/*.pem
 include bpsai_pair/security/Dockerfile.*
 recursive-include bpsai_pair/benchmarks/matrix/fixtures *.json
+# metis-reasoning fixtures are deliberately absent here: they embed
+# confidential portfolio content and must stay repo-only. Do not re-add.

{bpsai_pair-2.35.2/bpsai_pair.egg-info → bpsai_pair-2.37.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: bpsai-pair
-Version: 2.35.2
-Summary: AI-augmented pair programming framework with 270+ CLI commands for planning, orchestration, Trello/GitHub integration, and autonomous workflows
+Version: 2.37.0
+Summary: AI-augmented pair programming framework with 286 CLI commands for planning, orchestration, Trello/GitHub integration, and autonomous workflows
 Author-email: BPS AI Software <support@paircoder.ai>
 License-Expression: LicenseRef-Proprietary
 Project-URL: Homepage, https://paircoder.ai
@@ -23,63 +23,63 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer>=0.12
-Requires-Dist: rich>=13.7
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: ruamel.yaml>=0.18
-Requires-Dist: tiktoken>=0.5.0
-Requires-Dist: pydantic>=2.0
-Requires-Dist: cryptography>=41.0
-Requires-Dist: keyring>=25.0
-Requires-Dist: fastapi>=0.109.0
-Requires-Dist: jinja2>=3.1.0
-Requires-Dist: uvicorn>=0.27.0
-Requires-Dist: sse-starlette>=1.6.0
-Requires-Dist: toggl>=0.1.0
-Requires-Dist: anthropic>=0.76.0
-Requires-Dist: httpx>=0.27
-Requires-Dist: packaging>=21.0
-Requires-Dist: click>=8.0
-Requires-Dist: tenacity>=8.0
+Requires-Dist: typer<1.0,>=0.12
+Requires-Dist: rich<16.0,>=13.7
+Requires-Dist: pyyaml<7.0,>=6.0
+Requires-Dist: ruamel.yaml<1.0,>=0.18
+Requires-Dist: tiktoken<1.0,>=0.5.0
+Requires-Dist: pydantic<3.0,>=2.0
+Requires-Dist: cryptography<50.0,>=41.0
+Requires-Dist: keyring<26.0,>=25.0
+Requires-Dist: fastapi<1.0,>=0.109.0
+Requires-Dist: jinja2<4.0,>=3.1.0
+Requires-Dist: uvicorn<1.0,>=0.27.0
+Requires-Dist: sse-starlette<4.0,>=1.6.0
+Requires-Dist: toggl<1.0,>=0.1.0
+Requires-Dist: anthropic<1.0,>=0.76.0
+Requires-Dist: httpx<1.0,>=0.27
+Requires-Dist: packaging<27.0,>=21.0
+Requires-Dist: click<9.0,>=8.0
+Requires-Dist: tenacity<10.0,>=8.0
 Provides-Extra: test
-Requires-Dist: pytest>=8.0; extra == "test"
-Requires-Dist: pytest-cov>=4.0; extra == "test"
-Requires-Dist: pytest-asyncio>=0.21; extra == "test"
-Requires-Dist: pytest-timeout>=2.3; extra == "test"
-Requires-Dist: pytest-xdist>=3.5; extra == "test"
-Requires-Dist: respx>=0.21; extra == "test"
-Requires-Dist: py-trello>=0.19.0; extra == "test"
-Requires-Dist: tomli>=2.0; python_version < "3.11" and extra == "test"
+Requires-Dist: pytest<10.0,>=8.0; extra == "test"
+Requires-Dist: pytest-cov<8.0,>=4.0; extra == "test"
+Requires-Dist: pytest-asyncio<2.0,>=0.21; extra == "test"
+Requires-Dist: pytest-timeout<3.0,>=2.3; extra == "test"
+Requires-Dist: pytest-xdist<4.0,>=3.5; extra == "test"
+Requires-Dist: respx<1.0,>=0.21; extra == "test"
+Requires-Dist: py-trello<1.0,>=0.19.0; extra == "test"
+Requires-Dist: tomli<3.0,>=2.0; python_version < "3.11" and extra == "test"
 Provides-Extra: trello
-Requires-Dist: py-trello>=0.19.0; extra == "trello"
+Requires-Dist: py-trello<1.0,>=0.19.0; extra == "trello"
 Provides-Extra: github
-Requires-Dist: PyGithub>=2.1; extra == "github"
+Requires-Dist: PyGithub<3.0,>=2.1; extra == "github"
 Provides-Extra: timer
-Requires-Dist: toggl>=0.1.0; extra == "timer"
+Requires-Dist: toggl<1.0,>=0.1.0; extra == "timer"
 Provides-Extra: mcp
-Requires-Dist: mcp>=1.0.0; extra == "mcp"
+Requires-Dist: mcp<2.0,>=1.0.0; extra == "mcp"
 Provides-Extra: docs
-Requires-Dist: sphinx>=7.0; extra == "docs"
-Requires-Dist: sphinx-rtd-theme>=2.0; extra == "docs"
-Requires-Dist: sphinx-autodoc-typehints>=2.0; extra == "docs"
-Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
-Requires-Dist: myst-parser>=2.0; extra == "docs"
-Requires-Dist: sphinx-click>=5.0; extra == "docs"
+Requires-Dist: sphinx<10.0,>=7.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme<4.0,>=2.0; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints<4.0,>=2.0; extra == "docs"
+Requires-Dist: sphinx-copybutton<1.0,>=0.5; extra == "docs"
+Requires-Dist: myst-parser<6.0,>=2.0; extra == "docs"
+Requires-Dist: sphinx-click<7.0,>=5.0; extra == "docs"
 Provides-Extra: sandbox
-Requires-Dist: docker>=7.0.0; extra == "sandbox"
-Requires-Dist: dockerpty>=0.4.1; extra == "sandbox"
+Requires-Dist: docker<8.0,>=7.0.0; extra == "sandbox"
+Requires-Dist: dockerpty<1.0,>=0.4.1; extra == "sandbox"
 Provides-Extra: wizard
-Requires-Dist: anthropic>=0.76.0; extra == "wizard"
+Requires-Dist: anthropic<1.0,>=0.76.0; extra == "wizard"
 Provides-Extra: all
-Requires-Dist: py-trello>=0.19.0; extra == "all"
-Requires-Dist: mcp>=1.0.0; extra == "all"
-Requires-Dist: docker>=7.0.0; extra == "all"
-Requires-Dist: dockerpty>=0.4.1; extra == "all"
+Requires-Dist: py-trello<1.0,>=0.19.0; extra == "all"
+Requires-Dist: mcp<2.0,>=1.0.0; extra == "all"
+Requires-Dist: docker<8.0,>=7.0.0; extra == "all"
+Requires-Dist: dockerpty<1.0,>=0.4.1; extra == "all"
 Provides-Extra: integrations
-Requires-Dist: py-trello>=0.19.0; extra == "integrations"
+Requires-Dist: py-trello<1.0,>=0.19.0; extra == "integrations"
 Dynamic: license-file
 
-# PairCoder v2.35.2
+# PairCoder v2.37.0
 
 > AI-augmented pair programming framework with 270+ CLI commands
 
@@ -189,6 +189,19 @@ Check your license: `bpsai-pair license status`
 
 > Full release history and detailed notes: [paircoder.ai](https://paircoder.ai).
 
+**v2.37.0 — Opus 5, reliability, and honest benchmarks**
+- **Claude Opus 5, day-one support.** Registry, pricing, wizard picker, and a benchmark-panel row — while model routing deliberately stays on measured evidence, not release hype.
+- **A tier that still discriminates.** `hard-plus` fixtures restore separation above the saturating hard tier, so frontier models keep producing routing signal.
+- **Engage runs you can trust.** Resume actually skips completed tasks; a safe `cleanup-orphans` sweeper (dry-run first, preserved work excluded); a machine-level cap stops runaway concurrent runs from exhausting your machine.
+- **Benchmarks tell the truth.** Refusals are classified only on real metered attempts, privacy settings are honored end-to-end and fail closed, and a `matrix-regrade` command rebuilds reports from retained artifacts at zero API cost.
+
+**v2.36.0 — Benchmarks that discriminate, prices that reconcile**
+- **Difficulty tiers separate the field.** Fixtures now carry `easy`/`medium`/`hard` tiers with 10 new hard fixtures; reports score per tier, so models that all ace small tasks still rank meaningfully on hard ones.
+- **Conditions are first-class rows.** Pin thinking on/off and reasoning-effort per row (`model@effort`) and compare a model against itself, not just against others.
+- **One pricing source of truth.** Five divergent internal price tables are unified; OpenRouter prices auto-resolve from the live catalog, reasoning tokens are counted, and recorded costs reconcile against what providers actually bill.
+- **Sturdier autonomous runs.** Engage lands driver commits on the intended branch under worktree isolation, reports aborted runs honestly, guards each benchmark row against empty/null provider output, and the security-audit gate now reads the real branch diff.
+- **Faster CI, governed releases.** Python CI shards 4-way (~4-minute cycles); releases pin a hash manifest of the shipped `.claude/` payload so fleet upgrades auto-commit only on an exact match.
+
 **v2.35.2 — Benchmarks join reality, QC agent regains its browser**
 - **Benchmark fixtures speak your task types.** The matrix fixture set is re-keyed to the vocabulary real projects actually use (`feature`, `bugfix`, `chore`, `refactor`, `review`), with legacy strings alias-normalized at ingestion — so `benchmark matrix-run --auto-seed` primes the same calibration cells your production tasks read, instead of cells nothing joins.
 - **Fixtures can't over-demand.** A fail-closed fixture lint rejects any fixture whose grading requirements aren't explicitly stated in its own prompt, ending a class of false cross-model failures.

{bpsai_pair-2.35.2 → bpsai_pair-2.37.0}/README.md

@@ -1,4 +1,4 @@
-# PairCoder v2.35.2
+# PairCoder v2.37.0
 
 > AI-augmented pair programming framework with 270+ CLI commands
 
@@ -108,6 +108,19 @@ Check your license: `bpsai-pair license status`
 
 > Full release history and detailed notes: [paircoder.ai](https://paircoder.ai).
 
+**v2.37.0 — Opus 5, reliability, and honest benchmarks**
+- **Claude Opus 5, day-one support.** Registry, pricing, wizard picker, and a benchmark-panel row — while model routing deliberately stays on measured evidence, not release hype.
+- **A tier that still discriminates.** `hard-plus` fixtures restore separation above the saturating hard tier, so frontier models keep producing routing signal.
+- **Engage runs you can trust.** Resume actually skips completed tasks; a safe `cleanup-orphans` sweeper (dry-run first, preserved work excluded); a machine-level cap stops runaway concurrent runs from exhausting your machine.
+- **Benchmarks tell the truth.** Refusals are classified only on real metered attempts, privacy settings are honored end-to-end and fail closed, and a `matrix-regrade` command rebuilds reports from retained artifacts at zero API cost.
+
+**v2.36.0 — Benchmarks that discriminate, prices that reconcile**
+- **Difficulty tiers separate the field.** Fixtures now carry `easy`/`medium`/`hard` tiers with 10 new hard fixtures; reports score per tier, so models that all ace small tasks still rank meaningfully on hard ones.
+- **Conditions are first-class rows.** Pin thinking on/off and reasoning-effort per row (`model@effort`) and compare a model against itself, not just against others.
+- **One pricing source of truth.** Five divergent internal price tables are unified; OpenRouter prices auto-resolve from the live catalog, reasoning tokens are counted, and recorded costs reconcile against what providers actually bill.
+- **Sturdier autonomous runs.** Engage lands driver commits on the intended branch under worktree isolation, reports aborted runs honestly, guards each benchmark row against empty/null provider output, and the security-audit gate now reads the real branch diff.
+- **Faster CI, governed releases.** Python CI shards 4-way (~4-minute cycles); releases pin a hash manifest of the shipped `.claude/` payload so fleet upgrades auto-commit only on an exact match.
+
 **v2.35.2 — Benchmarks join reality, QC agent regains its browser**
 - **Benchmark fixtures speak your task types.** The matrix fixture set is re-keyed to the vocabulary real projects actually use (`feature`, `bugfix`, `chore`, `refactor`, `review`), with legacy strings alias-normalized at ingestion — so `benchmark matrix-run --auto-seed` primes the same calibration cells your production tasks read, instead of cells nothing joins.
 - **Fixtures can't over-demand.** A fail-closed fixture lint rejects any fixture whose grading requirements aren't explicitly stated in its own prompt, ending a class of false cross-model failures.

{bpsai_pair-2.35.2 → bpsai_pair-2.37.0}/bpsai_pair/benchmarks/matrix/__init__.py

@@ -23,10 +23,13 @@ Public API:
 - ``MatrixRow`` / ``MatrixResult`` / ``MatrixRunner`` -- 5x5 runner.
 - ``MATRIX_CSV_FIELDS`` -- documented column order for matrix.csv.
 - ``metered_call`` / ``CallOutcome`` -- provider dispatch for benchmark runs.
+- ``RowConditions`` / ``DEFAULT_CONDITIONS`` / ``cell_key`` -- per-row dispatch
+  conditions (thinking, reasoning effort) and their cell-key convention.
 - ``ModelConfig`` / ``default_models`` / ``filter_models`` -- model panel helpers.
 - ``estimate_cost`` -- provider-aware per-model USD/Mtok pricing.
-- ``aggregate_by_task_model`` / ``results_dir`` / ``write_matrix`` -- matrix
-  output helpers.
+- ``aggregate_by_task_model`` / ``aggregate_by_task_difficulty`` /
+  ``results_dir`` / ``write_matrix`` -- matrix output helpers.
+- ``DIFFICULTIES`` -- the three fixture difficulty tiers.
 """
 
 from __future__ import annotations
@@ -40,12 +43,16 @@ _LAZY_EXPORTS: dict[str, str] = {
     "check_lint_clean": "checks",
     "check_structural_sanity": "checks",
     "check_test_execution": "checks",
+    "cell_key": "conditions",
+    "DEFAULT_CONDITIONS": "conditions",
+    "RowConditions": "conditions",
     "CallOutcome": "dispatch",
     "metered_call": "dispatch",
     "MatrixResult": "engine",
-    "MatrixRow": "engine",
+    "MatrixRow": "row",
     "MatrixRunner": "engine",
     "aggregate_by_task_model": "engine",
+    "aggregate_by_task_difficulty": "engine",
     "quality_score_from_report": "engine",
     "MATRIX_CSV_FIELDS": "io",
     "results_dir": "io",
@@ -60,6 +67,7 @@ _LAZY_EXPORTS: dict[str, str] = {
     "aggregate": "scorer",
     "score": "scorer",
     "TASK_TYPES": "task_set",
+    "DIFFICULTIES": "task_set",
     "BenchmarkFixture": "task_set",
     "ExpectedShape": "task_set",
     "FixtureShapeMismatch": "task_set",

{bpsai_pair-2.35.2 → bpsai_pair-2.37.0}/bpsai_pair/benchmarks/matrix/checks.py

@@ -25,6 +25,72 @@ from bpsai_pair.benchmarks.matrix.scorer import CheckResult
 from bpsai_pair.benchmarks.matrix.task_set import BenchmarkFixture
 
 _TEST_DEF_RE = re.compile(r"^\s*def\s+test_\w+\s*\(", re.MULTILINE)
+_FENCE_RE = re.compile(r"```([^\n`]*)\n(.*?)```", re.DOTALL)
+
+# Extraction scans at most this much of a response. A code answer beyond
+# this is degenerate; the cap bounds regex/parse work on unbounded model
+# output (fences past the cap are simply not seen).
+_MAX_EXTRACT_CHARS = 1_000_000
+
+_PY_TAGS = frozenset({"python", "py", "python3"})
+
+
+def _parse(source: str) -> None:
+    """``ast.parse`` behind a seam so checks can catch pathological inputs
+    (RecursionError/MemoryError from adversarial nesting) as check
+    failures instead of crashing the whole matrix run."""
+    ast.parse(source)
+
+
+def _parses(source: str) -> bool:
+    """True when ``source`` parses; pathological inputs count as no."""
+    try:
+        _parse(source)
+    except (SyntaxError, RecursionError, MemoryError):
+        return False
+    return True
+
+
+def extract_python_source(output: str) -> str:
+    """Extract Python source from a model response for parsing/scanning.
+
+    If the output contains one or more fenced code blocks (```` ``` ```` or
+    ```` ```python ````), concatenate their contents in order -- models
+    sometimes split implementation and tests across two fences, so
+    concatenation lets a single ``ast.parse``/``def test_`` scan see both.
+    If the concatenation fails to parse, fall back to the single largest
+    fenced block. If there are no fences, return the output unchanged.
+    """
+    if not output:
+        return ""
+    # The fence regex's opening-line capture group already isolates the
+    # language tag (e.g. ```python) from the body, so no further stripping
+    # of a leading language-tag line is needed here.
+    fences = [
+        (lang.strip().lower(), body.strip("\n"))
+        for lang, body in _FENCE_RE.findall(output[:_MAX_EXTRACT_CHARS])
+    ]
+    blocks = [body for _lang, body in fences]
+    if not blocks:
+        return output
+
+    combined = "\n".join(blocks)
+    if _parses(combined):
+        return combined
+
+    # Concatenation failed. A malformed python-tagged fence must keep
+    # failing downstream — returning only a valid sibling block would
+    # score broken generated code as clean.
+    py_blocks = [body for lang, body in fences if lang in _PY_TAGS]
+    if any(not _parses(b) for b in py_blocks):
+        return combined
+    if py_blocks:
+        py_combined = "\n".join(py_blocks)
+        if _parses(py_combined):
+            return py_combined
+    # All python-tagged blocks parse individually (or none are tagged):
+    # the concat failure is a stitching artifact / non-Python fence.
+    return max(py_blocks or blocks, key=len)
 
 
 def check_test_execution(fixture: BenchmarkFixture, output: str) -> CheckResult:
@@ -35,11 +101,19 @@ def check_test_execution(fixture: BenchmarkFixture, output: str) -> CheckResult:
     name = "test_execution"
     if not fixture.expected_shape.produces_code:
         return CheckResult(name=name, status="skip", detail="not a code task")
+    if not output or not output.strip():
+        return CheckResult(name=name, status="fail", detail="empty model output")
+    source = extract_python_source(output)
     try:
-        ast.parse(output)
+        _parse(source)
     except SyntaxError as exc:
         return CheckResult(name=name, status="fail", detail=f"syntax error: {exc.msg}")
-    if not _TEST_DEF_RE.search(output):
+    except (RecursionError, MemoryError) as exc:
+        return CheckResult(
+            name=name, status="fail",
+            detail=f"pathological input: parse aborted ({type(exc).__name__})",
+        )
+    if not _TEST_DEF_RE.search(source):
         return CheckResult(
             name=name, status="fail", detail="no `def test_*` block found",
         )
@@ -54,10 +128,18 @@ def check_lint_clean(fixture: BenchmarkFixture, output: str) -> CheckResult:
     name = "lint_clean"
     if not fixture.expected_shape.produces_code:
         return CheckResult(name=name, status="skip", detail="not a code task")
+    if not output or not output.strip():
+        return CheckResult(name=name, status="fail", detail="empty model output")
+    source = extract_python_source(output)
     try:
-        ast.parse(output)
+        _parse(source)
     except SyntaxError as exc:
         return CheckResult(name=name, status="fail", detail=f"parse failed: {exc.msg}")
+    except (RecursionError, MemoryError) as exc:
+        return CheckResult(
+            name=name, status="fail",
+            detail=f"pathological input: parse aborted ({type(exc).__name__})",
+        )
     return CheckResult(name=name, status="pass")
 
 
@@ -67,8 +149,9 @@ def check_structural_sanity(
     """Verify output non-empty + constraint-conformance per ExpectedShape."""
     shape = fixture.expected_shape
     name = "structural_sanity"
+    output = output or ""
 
-    if len(output) < shape.min_length:
+    if len(output or "") < shape.min_length:
         return CheckResult(
             name=name,
             status="fail",

bpsai_pair-2.37.0/bpsai_pair/benchmarks/matrix/conditions.py

@@ -0,0 +1,84 @@
+"""Per-row dispatch conditions -- the condition-schema source of truth.
+
+A benchmark row is not fully described by ``(task_type, model_id)`` alone:
+the same model can be dispatched under different generation conditions
+(extended thinking on/off, a reasoning-effort level) and those conditions
+change what the row measures. ``RowConditions`` is the single schema for
+that axis:
+
+    {"thinking": "off" | "on", "reasoning_effort": <str | None>}
+
+``"off"``/``None`` is the default for both fields -- a row dispatched with
+no explicit condition override is indistinguishable from one built before
+this schema existed. That is deliberate: :func:`cell_key` returns the bare
+``model_id`` for default conditions so existing matrix cells stay joinable
+across a schema upgrade with no key migration. A non-default condition
+gets a deterministic ``@``-separated suffix instead -- ``fable@thinking``
+for thinking turned on, ``fable@effort-high`` for a non-default
+``reasoning_effort`` value, ``fable@thinking+effort-high`` when both are
+set. Any future condition field follows the same rule: default value ->
+no suffix contribution; non-default -> its own ``+``-joined segment,
+appended in field-declaration order.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+ThinkingMode = Literal["off", "on"]
+
+_VALID_THINKING_MODES: tuple[ThinkingMode, ...] = ("off", "on")
+
+
+@dataclass(frozen=True)
+class RowConditions:
+    """The dispatch conditions for one benchmark row. See module docstring."""
+
+    thinking: ThinkingMode = "off"
+    reasoning_effort: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.thinking not in _VALID_THINKING_MODES:
+            raise ValueError(
+                f"thinking must be one of {_VALID_THINKING_MODES}, "
+                f"got {self.thinking!r}",
+            )
+
+    def suffix(self) -> str:
+        """Deterministic ``+``-joined suffix for non-default fields (no
+        leading ``@``); empty string when every field is at its default."""
+        parts: list[str] = []
+        if self.thinking != "off":
+            parts.append("thinking")
+        if self.reasoning_effort:
+            parts.append(f"effort-{self.reasoning_effort}")
+        return "+".join(parts)
+
+    @property
+    def is_default(self) -> bool:
+        return not self.suffix()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"thinking": self.thinking, "reasoning_effort": self.reasoning_effort}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RowConditions":
+        data = data or {}
+        return cls(
+            thinking=data.get("thinking", "off"),
+            reasoning_effort=data.get("reasoning_effort"),
+        )
+
+
+DEFAULT_CONDITIONS = RowConditions()
+
+
+def cell_key(model_id: str, conditions: RowConditions) -> str:
+    """The matrix cell key for ``model_id`` under ``conditions``.
+
+    Bare ``model_id`` when ``conditions`` are all default; otherwise
+    ``model_id@<suffix>`` (see :meth:`RowConditions.suffix`).
+    """
+    suffix = conditions.suffix()
+    return model_id if not suffix else f"{model_id}@{suffix}"