opencode-arch 1.0.0__py3-none-any.whl

opencode_arch/learning/adapter.py

@@ -0,0 +1,157 @@
+"""Adaptive prompt optimization based on classified failure patterns.
+
+Queries historical telemetry for similar subsystems and applies learned
+strategies to modify prompt construction BEFORE the first attempt.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from opencode_arch.learning.patterns import (
+    FailureClassification,
+    PatternType,
+    Strategy,
+)
+
+
+@dataclass
+class PromptAdaptation:
+    """A specific adaptation to apply to prompt construction."""
+    strategy: Strategy
+    reason: str                          # why this adaptation is being applied
+    params: dict[str, Any] = field(default_factory=dict)
+    # params vary by strategy:
+    # EXPAND_DEP_CONTEXT: {"dep_modules": ["_base", "_config"], "include_body_hints": True}
+    # INCLUDE_SOURCE_EXCERPT: {"functions": ["complex_func"]}
+    # PRIORITIZE_CONSTANTS: {"boost_factor": 2.0}
+    # FIX_SIGNATURES: {"symbols": ["MyClass.method"]}
+    # INCREASE_CONTRACT_CAP: {"new_cap": 100}
+
+
+def get_adaptations(
+    subsystem_name: str,
+    dependency_count: int,
+    signature_count: int,
+    contract_count: int,
+    body_hint_coverage: float,
+    historical_patterns: list[dict[str, Any]] | None = None,
+) -> list[PromptAdaptation]:
+    """Determine prompt adaptations based on subsystem characteristics and history.
+
+    This is called BEFORE the first prompt attempt to proactively adjust
+    based on what we've learned from previous repos.
+
+    Args:
+        subsystem_name: Name of the current subsystem.
+        dependency_count: Number of upstream dependencies.
+        signature_count: Number of function signatures in model.
+        contract_count: Number of test contracts available.
+        body_hint_coverage: Fraction of functions with body_hints (0.0-1.0).
+        historical_patterns: Past failure patterns from telemetry for similar subsystems.
+
+    Returns:
+        List of adaptations to apply to prompt construction.
+    """
+    adaptations: list[PromptAdaptation] = []
+
+    # Rule 1: High dependency count → expand dep context proactively
+    if dependency_count >= 3:
+        adaptations.append(PromptAdaptation(
+            strategy=Strategy.EXPAND_DEP_CONTEXT,
+            reason=f"Subsystem has {dependency_count} dependencies (threshold: 3)",
+            params={"include_body_hints": True},
+        ))
+
+    # Rule 2: Low contract count → increase cap and prioritize what we have
+    if contract_count < 10:
+        adaptations.append(PromptAdaptation(
+            strategy=Strategy.INCREASE_CONTRACT_CAP,
+            reason=f"Only {contract_count} contracts available (low coverage)",
+            params={"new_cap": 200},  # Don't cap at all for low-contract subsystems
+        ))
+
+    # Rule 3: Low body_hint coverage → likely to fail on implementation details
+    if body_hint_coverage < 0.5 and signature_count > 5:
+        adaptations.append(PromptAdaptation(
+            strategy=Strategy.INCLUDE_SOURCE_EXCERPT,
+            reason=f"Body hint coverage is {body_hint_coverage:.0%} (threshold: 50%)",
+            params={"coverage_threshold": body_hint_coverage},
+        ))
+
+    # Rule 4: Historical patterns suggest specific strategies
+    if historical_patterns:
+        pattern_counts: dict[str, int] = {}
+        for entry in historical_patterns:
+            pattern = entry.get("pattern", "")
+            if pattern:
+                pattern_counts[pattern] = pattern_counts.get(pattern, 0) + 1
+
+        # If cross_dep failures dominate history for similar subsystems
+        if pattern_counts.get("cross_dep", 0) >= 2:
+            if not any(a.strategy == Strategy.EXPAND_DEP_CONTEXT for a in adaptations):
+                adaptations.append(PromptAdaptation(
+                    strategy=Strategy.EXPAND_DEP_CONTEXT,
+                    reason=f"Historical: {pattern_counts['cross_dep']} cross-dep failures in similar subsystems",
+                    params={"include_body_hints": True, "historical": True},
+                ))
+
+        # If wrong_constant failures are common
+        if pattern_counts.get("wrong_constant", 0) >= 2:
+            adaptations.append(PromptAdaptation(
+                strategy=Strategy.PRIORITIZE_CONSTANTS,
+                reason=f"Historical: {pattern_counts['wrong_constant']} constant mismatches in similar subsystems",
+                params={"boost_factor": 2.0},
+            ))
+
+    return adaptations
+
+
+def apply_adaptations(
+    adaptations: list[PromptAdaptation],
+    contract_cap: int = 50,
+    include_dep_body_hints: bool = False,
+) -> dict[str, Any]:
+    """Apply adaptations and return modified prompt parameters.
+
+    Returns a dict of overrides to pass to _build_prompt:
+    - contract_cap: int (max contracts to include)
+    - include_dep_body_hints: bool (include body_hints in dep context)
+    - extra_context: str (additional context to append)
+    - source_excerpts: list[str] (functions needing full source)
+    """
+    result: dict[str, Any] = {
+        "contract_cap": contract_cap,
+        "include_dep_body_hints": include_dep_body_hints,
+        "extra_context": "",
+        "source_excerpts": [],
+        "adaptations_applied": [],
+    }
+
+    for adaptation in adaptations:
+        result["adaptations_applied"].append(
+            f"{adaptation.strategy.value}: {adaptation.reason}"
+        )
+
+        if adaptation.strategy == Strategy.EXPAND_DEP_CONTEXT:
+            result["include_dep_body_hints"] = True
+
+        elif adaptation.strategy == Strategy.INCREASE_CONTRACT_CAP:
+            new_cap = adaptation.params.get("new_cap", 100)
+            result["contract_cap"] = max(result["contract_cap"], new_cap)
+
+        elif adaptation.strategy == Strategy.INCLUDE_SOURCE_EXCERPT:
+            functions = adaptation.params.get("functions", [])
+            result["source_excerpts"].extend(functions)
+
+        elif adaptation.strategy == Strategy.PRIORITIZE_CONSTANTS:
+            # Signal to put constants section BEFORE signatures in prompt
+            result["extra_context"] += "\n# NOTE: Constants are critical for this subsystem — use exact values.\n"
+
+        elif adaptation.strategy == Strategy.FIX_SIGNATURES:
+            symbols = adaptation.params.get("symbols", [])
+            if symbols:
+                result["extra_context"] += f"\n# IMPORTANT: Match exact signatures for: {', '.join(symbols)}\n"
+
+    return result

opencode_arch/learning/assessor.py

@@ -0,0 +1,170 @@
+"""Report card generation and grading for regen loop runs.
+
+Produces a self-assessment after each repo run: grade (A-F),
+trending metrics, failure pattern breakdown, and improvement actions.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ReportCard:
+    """Self-assessment report for a regen loop run."""
+    repo: str
+    mode: str  # "normal" or "blind"
+
+    # Core metrics
+    fidelity: float              # converged/total (0.0-1.0)
+    compression_ratio: float     # source_equiv / prompt_tokens
+    time_per_subsystem: float    # avg seconds
+    total_subsystems: int
+    converged_subsystems: int
+
+    # Trend (vs previous)
+    fidelity_trend: str = "STABLE"       # "UP", "DOWN", "STABLE"
+    compression_trend: str = "STABLE"
+
+    # Pattern breakdown
+    failure_patterns: dict[str, int] = field(default_factory=dict)
+    novel_patterns: int = 0
+
+    # Grade + actions
+    grade: str = "C"
+    improvement_actions: list[str] = field(default_factory=list)
+
+
+def _compute_grade(fidelity: float, compression_ratio: float, novel_patterns: int) -> str:
+    """Compute letter grade from metrics.
+
+    A: >90% fidelity, compression >5x, 0 novel patterns
+    B: >75% fidelity, compression >3x
+    C: >60% fidelity
+    D: >40% fidelity
+    F: <40% fidelity
+    """
+    if fidelity >= 0.9 and compression_ratio >= 5.0 and novel_patterns == 0:
+        return "A"
+    elif fidelity >= 0.75 and compression_ratio >= 3.0:
+        return "B"
+    elif fidelity >= 0.6:
+        return "C"
+    elif fidelity >= 0.4:
+        return "D"
+    else:
+        return "F"
+
+
+def _compute_trend(current: float, previous: float | None) -> str:
+    """Determine trend direction."""
+    if previous is None:
+        return "STABLE"
+    diff = current - previous
+    if diff > 0.05:
+        return "UP"
+    elif diff < -0.05:
+        return "DOWN"
+    return "STABLE"
+
+
+def _compute_improvement_actions(
+    fidelity: float,
+    compression_ratio: float,
+    failure_patterns: dict[str, int],
+    novel_patterns: int,
+) -> list[str]:
+    """Generate actionable improvement suggestions."""
+    actions = []
+
+    if fidelity < 0.7:
+        dominant_pattern = max(failure_patterns, key=failure_patterns.get) if failure_patterns else None
+        if dominant_pattern == "cross_dep":
+            actions.append("Expand dependency context: include body_hints for upstream modules")
+        elif dominant_pattern == "missing_impl":
+            actions.append("Increase body_hint detail level for complex functions")
+        elif dominant_pattern == "wrong_constant":
+            actions.append("Verify constant extraction captures all test-expected values")
+        else:
+            actions.append(f"Investigate dominant failure pattern: {dominant_pattern}")
+
+    if compression_ratio < 3.0:
+        actions.append("Model is not providing enough compression — review token allocation")
+
+    if novel_patterns > 0:
+        actions.append(f"Classify {novel_patterns} novel failure patterns and add to taxonomy")
+
+    if fidelity >= 0.9 and compression_ratio >= 5.0:
+        actions.append("System performing well — consider adding a new benchmark repo")
+
+    return actions
+
+
+def generate_report_card(
+    repo: str,
+    mode: str,
+    subsystem_results: dict[str, dict[str, Any]],
+    previous_fidelity: float | None = None,
+    previous_compression: float | None = None,
+) -> ReportCard:
+    """Generate a report card from regen loop results.
+
+    Args:
+        repo: Repository name.
+        mode: "normal" or "blind".
+        subsystem_results: Dict of {subsystem_name: result_dict} from run_regen_loop.
+        previous_fidelity: Fidelity from previous repo (for trend).
+        previous_compression: Compression from previous repo (for trend).
+
+    Returns:
+        ReportCard with grade, trends, and improvement actions.
+    """
+    total = len(subsystem_results)
+    converged = sum(1 for r in subsystem_results.values() if r.get("converged", False))
+    fidelity = converged / total if total > 0 else 0.0
+
+    # Compute avg compression from token metrics
+    compressions = []
+    times = []
+    for r in subsystem_results.values():
+        metrics = r.get("token_metrics", {})
+        if metrics.get("compression_ratio", 0) > 0:
+            compressions.append(metrics["compression_ratio"])
+        times.append(r.get("time_seconds", 0))
+
+    avg_compression = sum(compressions) / len(compressions) if compressions else 0.0
+    avg_time = sum(times) / len(times) if times else 0.0
+
+    # Aggregate failure patterns from classifications
+    failure_patterns: dict[str, int] = {}
+    novel = 0
+    for r in subsystem_results.values():
+        patterns = r.get("failure_patterns", {})
+        for pattern, count in patterns.items():
+            if pattern == "unknown":
+                novel += count
+            else:
+                failure_patterns[pattern] = failure_patterns.get(pattern, 0) + count
+
+    # Compute grade and trends
+    grade = _compute_grade(fidelity, avg_compression, novel)
+    fidelity_trend = _compute_trend(fidelity, previous_fidelity)
+    compression_trend = _compute_trend(avg_compression, previous_compression)
+    actions = _compute_improvement_actions(fidelity, avg_compression, failure_patterns, novel)
+
+    return ReportCard(
+        repo=repo,
+        mode=mode,
+        fidelity=fidelity,
+        compression_ratio=avg_compression,
+        time_per_subsystem=avg_time,
+        total_subsystems=total,
+        converged_subsystems=converged,
+        fidelity_trend=fidelity_trend,
+        compression_trend=compression_trend,
+        failure_patterns=failure_patterns,
+        novel_patterns=novel,
+        grade=grade,
+        improvement_actions=actions,
+    )

opencode_arch/learning/classifier.py

@@ -0,0 +1,144 @@
+"""Dual-level failure pattern classifier.
+
+Level 1 (Raw): Regex matching on pytest output text
+Level 2 (Structured): Parsed test results cross-referenced with model data
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from opencode_arch.learning.patterns import (
+    FailureClassification,
+    PatternType,
+    Strategy,
+    PATTERN_STRATEGIES,
+)
+
+
+# --- Level 1: Raw regex patterns ---
+
+_RAW_PATTERNS: list[tuple[str, PatternType, float]] = [
+    # CROSS_DEP: ImportError from project module
+    (r"ImportError: cannot import name '(\w+)' from '([^']+)'", PatternType.CROSS_DEP, 0.95),
+    (r"ModuleNotFoundError: No module named '(\w[\w.]*)'", PatternType.CROSS_DEP, 0.8),
+    (r"NameError: name '(\w+)' is not defined", PatternType.CROSS_DEP, 0.6),
+
+    # MISSING_IMPL: AttributeError on generated code
+    (r"AttributeError: (?:module |type object )?'(\w+)' has no attribute '(\w+)'", PatternType.MISSING_IMPL, 0.9),
+    (r"AttributeError: '(\w+)' object has no attribute '(\w+)'", PatternType.MISSING_IMPL, 0.9),
+
+    # WRONG_CONSTANT: Assertion with literal values
+    (r"AssertionError: assert ['\"](.+)['\"] == ['\"](.+)['\"]", PatternType.WRONG_CONSTANT, 0.85),
+    (r"AssertionError: assert (\d+) == (\d+)", PatternType.WRONG_CONSTANT, 0.85),
+    (r"AssertionError: (?:assert )?(.+) != (.+)", PatternType.WRONG_CONSTANT, 0.7),
+
+    # API_MISMATCH: Wrong number of arguments
+    (r"TypeError: (\w+)\(\) takes (\d+) positional argument", PatternType.API_MISMATCH, 0.9),
+    (r"TypeError: (\w+)\(\) got an unexpected keyword argument '(\w+)'", PatternType.API_MISMATCH, 0.9),
+    (r"TypeError: (\w+)\(\) missing (\d+) required positional", PatternType.API_MISMATCH, 0.9),
+
+    # TEST_INFRA: Test helper imports failing
+    (r"ModuleNotFoundError: No module named 'tests\.", PatternType.TEST_INFRA, 0.95),
+    (r"ModuleNotFoundError: No module named 'conftest'", PatternType.TEST_INFRA, 0.95),
+    (r"ImportError: cannot import name .+ from 'tests\.", PatternType.TEST_INFRA, 0.9),
+]
+
+
+def _classify_raw(test_output: str) -> list[FailureClassification]:
+    """Level 1: Classify failures from raw pytest output using regex."""
+    classifications: list[FailureClassification] = []
+    seen_patterns: set[tuple[PatternType, str]] = set()
+
+    for regex, pattern_type, confidence in _RAW_PATTERNS:
+        for match in re.finditer(regex, test_output):
+            # Deduplicate: same pattern + same primary symbol
+            key = (pattern_type, match.group(1) if match.groups() else "")
+            if key in seen_patterns:
+                continue
+            seen_patterns.add(key)
+
+            # Extract affected symbols
+            symbols = [g for g in match.groups() if g]
+
+            classifications.append(FailureClassification(
+                pattern=pattern_type,
+                confidence=confidence,
+                raw_signal=match.group(0),
+                structured_signal={"groups": symbols, "regex": regex},
+                suggested_strategy=PATTERN_STRATEGIES[pattern_type],
+                affected_symbols=symbols,
+            ))
+
+    return classifications
+
+
+def _classify_structured(
+    test_output: str,
+    pass_rate: float,
+    total_tests: int,
+    failed_tests: int,
+) -> list[FailureClassification]:
+    """Level 2: Structured classification from parsed test results."""
+    classifications: list[FailureClassification] = []
+
+    # COMPLEX_BEHAVIOR: Many failures in same test module suggest behavioral complexity
+    if failed_tests > 5 and pass_rate < 0.5:
+        classifications.append(FailureClassification(
+            pattern=PatternType.COMPLEX_BEHAVIOR,
+            confidence=0.7,
+            raw_signal=f"{failed_tests}/{total_tests} tests failed (pass_rate={pass_rate:.0%})",
+            structured_signal={
+                "failed_count": failed_tests,
+                "total_count": total_tests,
+                "pass_rate": pass_rate,
+            },
+            suggested_strategy=Strategy.INCREASE_CONTRACT_CAP,
+        ))
+
+    # If ALL tests fail with import errors, it's likely a single root cause
+    import_errors = len(re.findall(r"(?:Import|Module)Error", test_output))
+    if import_errors > 0 and import_errors >= failed_tests * 0.8:
+        # Most failures are import-related — likely a single cross-dep issue
+        classifications.append(FailureClassification(
+            pattern=PatternType.CROSS_DEP,
+            confidence=0.9,
+            raw_signal=f"{import_errors} import errors out of {failed_tests} failures",
+            structured_signal={"import_error_ratio": import_errors / max(failed_tests, 1)},
+            suggested_strategy=Strategy.EXPAND_DEP_CONTEXT,
+        ))
+
+    return classifications
+
+
+def classify_failures(
+    test_output: str,
+    pass_rate: float = 0.0,
+    total_tests: int = 0,
+    failed_tests: int = 0,
+) -> list[FailureClassification]:
+    """Classify test failures using both raw and structured analysis.
+
+    Args:
+        test_output: Raw pytest stdout/stderr text.
+        pass_rate: Overall pass rate (0.0-1.0).
+        total_tests: Total number of tests run.
+        failed_tests: Number of failed tests.
+
+    Returns:
+        List of classified failures, sorted by confidence (highest first).
+    """
+    raw_results = _classify_raw(test_output)
+    structured_results = _classify_structured(test_output, pass_rate, total_tests, failed_tests)
+
+    # Merge: structured adds context but don't duplicate pattern types
+    raw_patterns = {c.pattern for c in raw_results}
+    merged = list(raw_results)
+    for sc in structured_results:
+        if sc.pattern not in raw_patterns:
+            merged.append(sc)
+
+    # Sort by confidence descending
+    merged.sort(key=lambda c: c.confidence, reverse=True)
+    return merged

opencode_arch/learning/lessons.py

@@ -0,0 +1,139 @@
+"""Lesson extraction and storage.
+
+Extracts insights from regen loop outcomes that can inform future runs.
+Lessons are automatically derived from patterns and stored for retrieval.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Lesson:
+    """A learned insight from regen loop experience."""
+    lesson_id: str               # unique hash
+    discovered_repo: str         # where it was first observed
+    category: str                # "pattern", "optimization", "limitation", "success"
+    description: str             # human-readable insight
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+
+def _make_lesson_id(category: str, description: str) -> str:
+    """Generate a stable lesson ID from content."""
+    content = f"{category}:{description}"
+    return hashlib.sha256(content.encode()).hexdigest()[:12]
+
+
+def extract_lessons(
+    repo: str,
+    mode: str,
+    subsystem_results: dict[str, dict[str, Any]],
+) -> list[Lesson]:
+    """Extract lessons from a completed regen loop run.
+
+    Looks for:
+    - Correlation between features and convergence
+    - Novel patterns not in taxonomy
+    - Success patterns worth replicating
+    - Limitations to document
+
+    Args:
+        repo: Repository name.
+        mode: "normal" or "blind".
+        subsystem_results: Results from run_regen_loop.
+
+    Returns:
+        List of lessons extracted.
+    """
+    lessons: list[Lesson] = []
+
+    # Analyze convergence patterns
+    converged_features: list[dict] = []
+    failed_features: list[dict] = []
+
+    for name, result in subsystem_results.items():
+        features = result.get("features", {})
+        if result.get("converged", False):
+            converged_features.append(features)
+        else:
+            failed_features.append(features)
+
+    # Lesson: Contract count threshold
+    if converged_features and failed_features:
+        avg_conv_contracts = sum(f.get("contract_count", 0) for f in converged_features) / len(converged_features)
+        avg_fail_contracts = sum(f.get("contract_count", 0) for f in failed_features) / len(failed_features)
+
+        if avg_conv_contracts > avg_fail_contracts * 2:
+            desc = (f"Subsystems with >{int(avg_fail_contracts)} contracts converge "
+                    f"({avg_conv_contracts:.0f} avg vs {avg_fail_contracts:.0f} avg)")
+            lessons.append(Lesson(
+                lesson_id=_make_lesson_id("pattern", desc),
+                discovered_repo=repo,
+                category="pattern",
+                description=desc,
+                evidence={
+                    "avg_converged_contracts": avg_conv_contracts,
+                    "avg_failed_contracts": avg_fail_contracts,
+                    "mode": mode,
+                },
+            ))
+
+    # Lesson: Signature count correlation
+    if converged_features and failed_features:
+        avg_conv_sigs = sum(f.get("signature_count", 0) for f in converged_features) / len(converged_features)
+        avg_fail_sigs = sum(f.get("signature_count", 0) for f in failed_features) / len(failed_features)
+
+        if avg_fail_sigs > avg_conv_sigs * 1.5:
+            desc = (f"High signature count ({avg_fail_sigs:.0f}) correlates with failure — "
+                    "complex modules need more context")
+            lessons.append(Lesson(
+                lesson_id=_make_lesson_id("limitation", desc),
+                discovered_repo=repo,
+                category="limitation",
+                description=desc,
+                evidence={
+                    "avg_converged_signatures": avg_conv_sigs,
+                    "avg_failed_signatures": avg_fail_sigs,
+                },
+            ))
+
+    # Lesson: Perfect fidelity on simple subsystems
+    simple_converged = [
+        name for name, r in subsystem_results.items()
+        if r.get("converged") and r.get("features", {}).get("signature_count", 0) < 10
+    ]
+    if len(simple_converged) >= 3:
+        desc = f"Simple subsystems (<10 signatures) reliably converge: {', '.join(simple_converged[:5])}"
+        lessons.append(Lesson(
+            lesson_id=_make_lesson_id("success", desc),
+            discovered_repo=repo,
+            category="success",
+            description=desc,
+            evidence={"subsystems": simple_converged},
+        ))
+
+    # Lesson: All failures are same pattern (systemic issue)
+    all_patterns: dict[str, int] = {}
+    for r in subsystem_results.values():
+        if not r.get("converged"):
+            for pattern, count in r.get("failure_patterns", {}).items():
+                all_patterns[pattern] = all_patterns.get(pattern, 0) + count
+
+    if all_patterns:
+        dominant = max(all_patterns, key=all_patterns.get)
+        total_failures = sum(all_patterns.values())
+        if all_patterns[dominant] / total_failures > 0.7:
+            desc = (f"Dominant failure pattern '{dominant}' accounts for "
+                    f"{all_patterns[dominant]}/{total_failures} failures — systemic issue")
+            lessons.append(Lesson(
+                lesson_id=_make_lesson_id("pattern", desc),
+                discovered_repo=repo,
+                category="pattern",
+                description=desc,
+                evidence={"pattern_distribution": all_patterns},
+            ))
+
+    return lessons