asft 0.1.0__py3-none-any.whl

asft/__init__.py

@@ -0,0 +1,30 @@
+"""
+ASFT — Adaptive Sparse Fine-Tuning Framework
+=============================================
+
+A next-generation, hardware-adaptive AI learning framework that dramatically
+reduces training cost and time while improving accuracy, reliability, and
+reasoning quality.
+
+Learning Priority Hierarchy:
+    Memory → Workflow Optimization → Tool Learning →
+    Skill Packs → Sparse Fine-Tuning → Full Fine-Tuning
+
+Full retraining is always the last resort.
+"""
+
+__version__ = "0.1.0"
+__author__ = "ASFT Contributors"
+__license__ = "MIT"
+
+from asft.core.config import ASFTConfig
+from asft.core.hardware_profiler import HardwareProfile, detect_hardware
+from asft.core.registry import Registry
+
+__all__ = [
+    "__version__",
+    "ASFTConfig",
+    "detect_hardware",
+    "HardwareProfile",
+    "Registry",
+]

asft/accuracy/__init__.py

@@ -0,0 +1,16 @@
+"""ASFT Accuracy Package."""
+
+from asft.accuracy.confidence_scorer import ConfidenceScore, ConfidenceScorer
+from asft.accuracy.multi_pass_reasoner import MultiPassReasoner, ReasoningResult
+from asft.accuracy.self_critique import CritiqueResult, SelfCritiqueEngine
+from asft.accuracy.verification_layer import VerificationLayer
+
+__all__ = [
+    "ConfidenceScorer",
+    "ConfidenceScore",
+    "MultiPassReasoner",
+    "ReasoningResult",
+    "SelfCritiqueEngine",
+    "CritiqueResult",
+    "VerificationLayer",
+]

asft/accuracy/confidence_scorer.py

@@ -0,0 +1,167 @@
+"""
+Confidence Scorer — Assigns confidence, reliability, and verification scores
+to every model output. Low scores trigger additional reasoning passes.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConfidenceScore:
+    confidence: float  # 0–1: overall output confidence
+    reliability: float  # 0–1: factual reliability estimate
+    verification: float  # 0–1: how well the output can be verified
+    composite: float  # weighted combination
+    flags: list[str]  # detected quality issues
+
+    @property
+    def needs_extra_pass(self) -> bool:
+        return self.composite < 0.7
+
+    @property
+    def label(self) -> str:
+        if self.composite >= 0.85:
+            return "HIGH"
+        elif self.composite >= 0.65:
+            return "MEDIUM"
+        else:
+            return "LOW"
+
+    def __str__(self) -> str:
+        return (
+            f"ConfidenceScore [{self.label}] "
+            f"conf={self.confidence:.2f} rel={self.reliability:.2f} "
+            f"ver={self.verification:.2f} composite={self.composite:.2f}"
+            + (f" flags={self.flags}" if self.flags else "")
+        )
+
+
+# Patterns that indicate uncertainty or low confidence
+_UNCERTAINTY_PATTERNS = [
+    r"\bi (think|believe|guess|assume)\b",
+    r"\b(probably|possibly|maybe|perhaps|might|could be)\b",
+    r"\b(i('m| am) not sure|i don't know|uncertain|unclear)\b",
+    r"\b(approximately|roughly|around|about)\b",
+]
+
+# Patterns that indicate hallucination risk
+_HALLUCINATION_PATTERNS = [
+    r"\bspecific(ally)? (in|on|at) \d{4}\b",  # specific years without context
+    r"\baccording to (a|the) (recent|new) study\b",
+    r"\bexperts (say|agree|claim)\b",
+    r"\bit('s| is) (widely|commonly|generally) known\b",
+]
+
+# Patterns that indicate verifiable content
+_VERIFIABLE_PATTERNS = [
+    r"\d+(\.\d+)?",  # numbers
+    r"```[\s\S]+?```",  # code blocks
+    r"https?://\S+",  # URLs
+    r"\b\d{4}\b",  # years
+]
+
+
+class ConfidenceScorer:
+    """
+    Multi-dimensional output quality scorer.
+
+    Analyzes:
+      - Linguistic uncertainty markers
+      - Hallucination risk patterns
+      - Verifiability signals
+      - Output length and structure
+      - Task-specific quality indicators
+    """
+
+    def __init__(self, threshold: float = 0.7):
+        self._threshold = threshold
+        self._uncertainty_re = [re.compile(p, re.IGNORECASE) for p in _UNCERTAINTY_PATTERNS]
+        self._hallucination_re = [re.compile(p, re.IGNORECASE) for p in _HALLUCINATION_PATTERNS]
+        self._verifiable_re = [re.compile(p) for p in _VERIFIABLE_PATTERNS]
+
+    def score(
+        self, output: str, task_type: str = "general", model_logprobs: list[float] | None = None
+    ) -> ConfidenceScore:
+        """Score a model output."""
+        flags: list[str] = []
+
+        if not output or len(output.strip()) < 5:
+            return ConfidenceScore(0.1, 0.1, 0.1, 0.1, ["empty_output"])
+
+        # 1. Confidence: based on uncertainty language
+        confidence = self._score_confidence(output, flags)
+
+        # 2. Reliability: based on hallucination risk patterns
+        reliability = self._score_reliability(output, flags)
+
+        # 3. Verification: how verifiable is the output
+        verification = self._score_verification(output)
+
+        # 4. Length/structure bonus
+        length_bonus = min(0.1, len(output) / 2000)
+        has_structure = any(c in output for c in ["\n", "##", "- ", "1."])
+        structure_bonus = 0.05 if has_structure else 0.0
+
+        # 5. Log-prob bonus if available
+        logprob_bonus = 0.0
+        if model_logprobs:
+            avg_logprob = sum(model_logprobs) / len(model_logprobs)
+            logprob_bonus = min(0.1, max(0.0, (avg_logprob + 5) / 50))
+
+        # Composite (weighted)
+        composite = (
+            0.40 * confidence
+            + 0.30 * reliability
+            + 0.20 * verification
+            + length_bonus
+            + structure_bonus
+            + logprob_bonus
+        )
+        composite = min(1.0, max(0.0, composite))
+
+        return ConfidenceScore(
+            confidence=round(confidence, 3),
+            reliability=round(reliability, 3),
+            verification=round(verification, 3),
+            composite=round(composite, 3),
+            flags=flags,
+        )
+
+    def _score_confidence(self, text: str, flags: list[str]) -> float:
+        uncertainty_hits = sum(1 for p in self._uncertainty_re if p.search(text))
+        if uncertainty_hits >= 3:
+            flags.append("high_uncertainty")
+            return 0.4
+        elif uncertainty_hits >= 1:
+            flags.append("some_uncertainty")
+            return 0.7
+        return 0.9
+
+    def _score_reliability(self, text: str, flags: list[str]) -> float:
+        hallucination_hits = sum(1 for p in self._hallucination_re if p.search(text))
+        if hallucination_hits >= 2:
+            flags.append("hallucination_risk")
+            return 0.35
+        elif hallucination_hits >= 1:
+            flags.append("minor_hallucination_risk")
+            return 0.65
+        return 0.85
+
+    def _score_verification(self, text: str) -> float:
+        verifiable_hits = sum(1 for p in self._verifiable_re if p.search(text))
+        return min(1.0, 0.3 + verifiable_hits * 0.15)
+
+    def batch_score(self, outputs: list[str], task_type: str = "general") -> list[ConfidenceScore]:
+        return [self.score(o, task_type) for o in outputs]
+
+    def best_output(self, outputs: list[str], task_type: str = "general") -> tuple:
+        """Select the highest-confidence output from a list."""
+        scores = self.batch_score(outputs, task_type)
+        best_idx = max(range(len(scores)), key=lambda i: scores[i].composite)
+        return outputs[best_idx], scores[best_idx]

asft/accuracy/multi_pass_reasoner.py

@@ -0,0 +1,185 @@
+"""
+Multi-Pass Reasoner — Generates K candidate solutions, scores each,
+and returns the highest-confidence answer. Implements self-consistency reasoning.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from asft.accuracy.confidence_scorer import ConfidenceScore, ConfidenceScorer
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ReasoningResult:
+    best_output: str
+    best_score: ConfidenceScore
+    all_outputs: list[str] = field(default_factory=list)
+    all_scores: list[ConfidenceScore] = field(default_factory=list)
+    consensus_output: str | None = None
+    passes_used: int = 1
+    task_type: str = "general"
+
+    def summary(self) -> str:
+        return (
+            f"MultiPassReasoning: {self.passes_used} passes | "
+            f"best={self.best_score.composite:.3f} | "
+            f"task={self.task_type}"
+        )
+
+
+class MultiPassReasoner:
+    """
+    Generates multiple candidate solutions and selects the best one.
+
+    Strategies:
+      - best_of_k:       Generate K outputs, pick highest-confidence
+      - self_consistency: Pick output with highest agreement across candidates
+      - escalating:       Start with 1 pass, escalate if confidence is low
+    """
+
+    def __init__(
+        self,
+        k: int = 3,
+        min_confidence: float = 0.7,
+        strategy: str = "best_of_k",
+    ):
+        self._k = k
+        self._min_confidence = min_confidence
+        self._strategy = strategy
+        self._scorer = ConfidenceScorer(threshold=min_confidence)
+
+    def reason(
+        self,
+        generate_fn: Callable[[int], list[str]],
+        task_type: str = "general",
+    ) -> ReasoningResult:
+        """
+        Run multi-pass reasoning.
+
+        Args:
+            generate_fn: callable(n_samples) → List[str] — model generation function
+            task_type: domain hint for scoring
+
+        Returns:
+            ReasoningResult with best output and all candidates
+        """
+        if self._strategy == "escalating":
+            return self._escalating_pass(generate_fn, task_type)
+        elif self._strategy == "self_consistency":
+            return self._self_consistency(generate_fn, task_type)
+        else:
+            return self._best_of_k(generate_fn, task_type)
+
+    def _best_of_k(self, generate_fn, task_type: str) -> ReasoningResult:
+        """Generate K outputs, score all, return highest-confidence."""
+        outputs = generate_fn(self._k)
+        if not outputs:
+            return ReasoningResult(
+                best_output="", best_score=ConfidenceScore(0, 0, 0, 0, ["no_output"])
+            )
+
+        scores = self._scorer.batch_score(outputs, task_type)
+        best_idx = max(range(len(scores)), key=lambda i: scores[i].composite)
+
+        logger.debug(
+            "BestOfK[%d]: scores=%s best=%.3f",
+            self._k,
+            [round(s.composite, 3) for s in scores],
+            scores[best_idx].composite,
+        )
+
+        return ReasoningResult(
+            best_output=outputs[best_idx],
+            best_score=scores[best_idx],
+            all_outputs=outputs,
+            all_scores=scores,
+            passes_used=len(outputs),
+            task_type=task_type,
+        )
+
+    def _self_consistency(self, generate_fn, task_type: str) -> ReasoningResult:
+        """Generate K outputs, find the one with most agreement."""
+        outputs = generate_fn(self._k)
+        if not outputs:
+            return ReasoningResult(
+                best_output="", best_score=ConfidenceScore(0, 0, 0, 0, ["no_output"])
+            )
+
+        scores = self._scorer.batch_score(outputs, task_type)
+
+        # Find consensus: output most similar (by token overlap) to others
+        consensus_idx = self._find_consensus(outputs)
+        consensus = outputs[consensus_idx]
+
+        best_idx = max(range(len(scores)), key=lambda i: scores[i].composite)
+
+        return ReasoningResult(
+            best_output=outputs[best_idx],
+            best_score=scores[best_idx],
+            all_outputs=outputs,
+            all_scores=scores,
+            consensus_output=consensus,
+            passes_used=len(outputs),
+            task_type=task_type,
+        )
+
+    def _escalating_pass(self, generate_fn, task_type: str) -> ReasoningResult:
+        """
+        Start with 1 pass. If confidence is below threshold, escalate to K.
+        Minimizes compute for easy tasks.
+        """
+        # First pass
+        outputs = generate_fn(1)
+        if not outputs:
+            return ReasoningResult(
+                best_output="", best_score=ConfidenceScore(0, 0, 0, 0, ["no_output"])
+            )
+
+        score = self._scorer.score(outputs[0], task_type)
+        if score.composite >= self._min_confidence:
+            logger.debug("Escalating: single pass sufficient (score=%.3f)", score.composite)
+            return ReasoningResult(
+                best_output=outputs[0],
+                best_score=score,
+                all_outputs=outputs,
+                all_scores=[score],
+                passes_used=1,
+                task_type=task_type,
+            )
+
+        # Escalate: generate K-1 more
+        logger.debug(
+            "Escalating: low confidence (%.3f) — generating %d more", score.composite, self._k - 1
+        )
+        extra = generate_fn(self._k - 1)
+        all_outputs = outputs + extra
+        all_scores = self._scorer.batch_score(all_outputs, task_type)
+        best_idx = max(range(len(all_scores)), key=lambda i: all_scores[i].composite)
+
+        return ReasoningResult(
+            best_output=all_outputs[best_idx],
+            best_score=all_scores[best_idx],
+            all_outputs=all_outputs,
+            all_scores=all_scores,
+            passes_used=len(all_outputs),
+            task_type=task_type,
+        )
+
+    def _find_consensus(self, outputs: list[str]) -> int:
+        """Find the output with highest token overlap to all others."""
+
+        def token_overlap(a: str, b: str) -> float:
+            ta, tb = set(a.lower().split()), set(b.lower().split())
+            return len(ta & tb) / max(1, len(ta | tb))
+
+        scores = []
+        for i, out in enumerate(outputs):
+            total = sum(token_overlap(out, outputs[j]) for j in range(len(outputs)) if j != i)
+            scores.append(total)
+
+        return int(max(range(len(scores)), key=lambda i: scores[i]))

asft/accuracy/self_critique.py

@@ -0,0 +1,187 @@
+"""
+Self-Critique Engine — Reviews model outputs before finalizing.
+Detects logical errors, contradictions, hallucinations, and weak reasoning.
+Automatically revises answers when issues are detected.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CritiqueResult:
+    original_output: str
+    revised_output: str
+    issues_found: list[str]
+    was_revised: bool
+    critique_rounds: int
+    quality_improvement: float  # estimated improvement 0–1
+
+    @property
+    def is_clean(self) -> bool:
+        return len(self.issues_found) == 0
+
+
+class SelfCritiqueEngine:
+    """
+    Reviews and revises model outputs to detect and fix:
+      - Logical errors and contradictions
+      - Hallucination indicators
+      - Missing information
+      - Weak or unsupported reasoning
+      - Inconsistent statements
+
+    Uses a generate_fn (model inference callable) to produce revisions.
+    """
+
+    def __init__(self, max_rounds: int = 2):
+        self._max_rounds = max_rounds
+        self._issue_detectors = [
+            self._detect_contradictions,
+            self._detect_hallucination_markers,
+            self._detect_logical_gaps,
+            self._detect_incomplete_response,
+        ]
+
+    def critique(
+        self,
+        output: str,
+        original_task: str,
+        generate_fn: Callable[[str], str] | None = None,
+    ) -> CritiqueResult:
+        """
+        Critique an output and optionally revise it.
+
+        Args:
+            output: the original model output to critique
+            original_task: the original task/question
+            generate_fn: optional callable(prompt) → str for generating revisions
+
+        Returns:
+            CritiqueResult with original, revised, and issue list
+        """
+        current = output
+        all_issues: list[str] = []
+        rounds = 0
+        was_revised = False
+
+        for round_num in range(self._max_rounds):
+            issues = self._detect_issues(current)
+            if not issues:
+                logger.debug("SelfCritique round %d: no issues found", round_num)
+                break
+
+            all_issues.extend(issues)
+            logger.info(
+                "SelfCritique round %d: %d issues found: %s", round_num, len(issues), issues
+            )
+
+            if generate_fn is None:
+                # No model available — mark issues but cannot revise
+                break
+
+            # Generate a revised version
+            revision_prompt = self._build_revision_prompt(
+                original_task=original_task,
+                current_output=current,
+                issues=issues,
+            )
+            revised = generate_fn(revision_prompt)
+            if revised and revised.strip() and revised != current:
+                current = revised
+                was_revised = True
+                rounds += 1
+            else:
+                break
+
+        quality_improvement = min(0.3, 0.1 * rounds) if was_revised else 0.0
+
+        return CritiqueResult(
+            original_output=output,
+            revised_output=current,
+            issues_found=list(set(all_issues)),
+            was_revised=was_revised,
+            critique_rounds=rounds,
+            quality_improvement=quality_improvement,
+        )
+
+    def _detect_issues(self, text: str) -> list[str]:
+        issues = []
+        for detector in self._issue_detectors:
+            found = detector(text)
+            if found:
+                issues.extend(found)
+        return issues
+
+    def _detect_contradictions(self, text: str) -> list[str]:
+        """Detect obvious contradictions using negation patterns."""
+        sentences = [s.strip() for s in re.split(r"[.!?]", text) if len(s.strip()) > 10]
+        issues = []
+        for i in range(len(sentences)):
+            for j in range(i + 1, min(i + 5, len(sentences))):
+                a, b = sentences[i].lower(), sentences[j].lower()
+                # Simple: check if one sentence negates a claim from another
+                a_words = set(a.split())
+                b_words = set(b.split())
+                shared = a_words & b_words
+                # Check if "not" appears near shared words
+                if len(shared) > 3 and ("not" in b_words) != ("not" in a_words):
+                    issues.append(f"potential_contradiction: '{sentences[i][:40]}...'")
+                    break
+        return issues[:2]  # Cap at 2 to avoid noise
+
+    def _detect_hallucination_markers(self, text: str) -> list[str]:
+        """Detect patterns commonly associated with hallucinations."""
+        patterns = [
+            (r"\brecently published\b.*\bstudy\b", "unsourced_study_claim"),
+            (r"\bexperts agree\b", "vague_expert_claim"),
+            (r"\b(all|every|always|never)\b.*\b(are|is|do|does)\b", "absolute_claim"),
+            (r"\bscientifically proven\b", "unverified_scientific_claim"),
+        ]
+        issues = []
+        for pattern, label in patterns:
+            if re.search(pattern, text, re.IGNORECASE):
+                issues.append(label)
+        return issues
+
+    def _detect_logical_gaps(self, text: str) -> list[str]:
+        """Detect reasoning gaps — conclusions without supporting reasoning."""
+        issues = []
+        has_conclusion = any(
+            w in text.lower() for w in ["therefore", "thus", "hence", "so", "conclusion"]
+        )
+        has_reasoning = any(
+            w in text.lower() for w in ["because", "since", "given that", "due to", "as a result"]
+        )
+        if has_conclusion and not has_reasoning and len(text) > 100:
+            issues.append("conclusion_without_reasoning")
+        return issues
+
+    def _detect_incomplete_response(self, text: str) -> list[str]:
+        """Detect suspiciously short or abruptly cut-off responses."""
+        issues = []
+        if len(text.strip()) < 20:
+            issues.append("response_too_short")
+        if text.strip() and text.strip()[-1] not in ".!?\"'`":
+            if len(text) > 50:  # Not just a very short answer
+                issues.append("response_appears_truncated")
+        return issues
+
+    def _build_revision_prompt(
+        self, original_task: str, current_output: str, issues: list[str]
+    ) -> str:
+        issues_str = "\n".join(f"  - {issue}" for issue in issues)
+        return (
+            f"Review and improve this response. The following issues were detected:\n"
+            f"{issues_str}\n\n"
+            f"Original task: {original_task}\n\n"
+            f"Current response:\n{current_output}\n\n"
+            f"Provide an improved response that fixes the identified issues. "
+            f"Be accurate, clear, and complete."
+        )