algo-cli-runtime 0.14.0__py3-none-any.whl

algo_cli/intelligence/content_extractor.py

@@ -0,0 +1,132 @@
+"""B61. Content Extraction Pipeline (Trafilatura Pattern).
+
+HTML → clean text/markdown with metadata, sitemap/feed discovery, URL dedup.
+Source: trafilatura pattern.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Iterable
+
+
+class ContentType(Enum):
+    HTML = auto()
+    MARKDOWN = auto()
+    PLAINTEXT = auto()
+    PDF = auto()
+    JSON = auto()
+    UNKNOWN = auto()
+
+
+@dataclass
+class ExtractedContent:
+    title: str = ""
+    text: str = ""
+    author: str = ""
+    date: str = ""
+    url: str = ""
+    content_type: ContentType = ContentType.PLAINTEXT
+    paragraphs: list[str] = field(default_factory=list)
+    links: list[str] = field(default_factory=list)
+    metadata: dict[str, str] = field(default_factory=dict)
+    word_count: int = 0
+
+
+class ContentExtractor:
+    """Extract clean text and metadata from HTML."""
+
+    # Tags to strip entirely
+    STRIP_TAGS = {"script", "style", "nav", "footer", "header", "aside", "noscript", "iframe"}
+    # Tags that contain main content
+    CONTENT_TAGS = {"article", "main", "section", "div"}
+    # Block-level tags
+    BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "pre", "td", "th"}
+
+    def extract(self, html: str, url: str = "") -> ExtractedContent:
+        result = ExtractedContent(url=url, content_type=ContentType.HTML)
+
+        # Extract title
+        title_match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+        if title_match:
+            result.title = title_match.group(1).strip()
+
+        # Extract metadata
+        for match in re.finditer(r'<meta\s+(?:name|property)=["\']([^"\']+)["\']\s+content=["\']([^"\']*)["\']', html, re.IGNORECASE):
+            result.metadata[match.group(1)] = match.group(2)
+            if match.group(1).lower() == "author":
+                result.author = match.group(2)
+            if match.group(1).lower() in ("date", "article:published_time", "publish_date"):
+                result.date = match.group(2)
+
+        # Strip unwanted tags
+        cleaned = html
+        for tag in self.STRIP_TAGS:
+            cleaned = re.sub(rf"<{tag}[^>]*>.*?</{tag}>", "", cleaned, flags=re.IGNORECASE | re.DOTALL)
+
+        # Extract links
+        for match in re.finditer(r'href=["\']([^"\']+)["\']', cleaned, re.IGNORECASE):
+            result.links.append(match.group(1))
+
+        # Convert to text
+        text = cleaned
+        # Replace block tags with newlines
+        for tag in self.BLOCK_TAGS:
+            text = re.sub(rf"</?{tag}[^>]*>", "\n", text, flags=re.IGNORECASE)
+        # Replace <br> with newline
+        text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
+        # Strip all remaining tags
+        text = re.sub(r"<[^>]+>", "", text)
+        # Decode entities
+        text = text.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", '"').replace("&#39;", "'").replace("&nbsp;", " ")
+        # Normalize whitespace
+        text = re.sub(r"\n\s*\n", "\n\n", text).strip()
+
+        result.text = text
+        result.paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+        result.word_count = len(text.split())
+        return result
+
+    def to_markdown(self, content: ExtractedContent) -> str:
+        """Convert extracted content to markdown."""
+        lines: list[str] = []
+        if content.title:
+            lines.append(f"# {content.title}")
+            lines.append("")
+        if content.author:
+            lines.append(f"*By {content.author}*")
+        if content.date:
+            lines.append(f"*{content.date}*")
+        if content.url:
+            lines.append(f"Source: {content.url}")
+        lines.append("")
+        for para in content.paragraphs:
+            lines.append(para)
+            lines.append("")
+        return "\n".join(lines).strip()
+
+
+class URLDeduplicator:
+    """Track seen URLs to avoid re-fetching."""
+
+    def __init__(self) -> None:
+        self._seen: set[str] = set()
+
+    def is_new(self, url: str) -> bool:
+        normalized = self._normalize(url)
+        if normalized in self._seen:
+            return False
+        self._seen.add(normalized)
+        return True
+
+    def filter_new(self, urls: Iterable[str]) -> list[str]:
+        return [u for u in urls if self.is_new(u)]
+
+    @staticmethod
+    def _normalize(url: str) -> str:
+        url = url.lower().rstrip("/")
+        url = re.sub(r"^https?://", "", url)
+        url = re.sub(r"www\.", "", url)
+        url = url.split("#")[0]
+        return url

algo_cli/intelligence/context_adaptive.py

@@ -0,0 +1,102 @@
+"""H21 — Context-Adaptive Parameter Selection.
+
+Classify context → select parameters before execution.
+Mined from G0DM0D3 PAPER.md §3.2 AutoTune.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ContextPattern:
+    """A pattern that matches a context and maps to parameters."""
+
+    name: str
+    keywords: list[str]
+    weight: float = 1.0
+    parameters: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ContextClassification:
+    """Result of classifying a context."""
+
+    matched_patterns: list[str] = field(default_factory=list)
+    confidence: float = 0.0
+    selected_parameters: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "matched_patterns": list(self.matched_patterns),
+            "confidence": self.confidence,
+            "selected_parameters": dict(self.selected_parameters),
+        }
+
+
+class ContextAdaptiveSelector:
+    """Score context against patterns and interpolate parameters."""
+
+    def __init__(self) -> None:
+        self._patterns: list[ContextPattern] = []
+        self._defaults: dict[str, Any] = {}
+
+    def set_defaults(self, params: dict[str, Any]) -> None:
+        self._defaults = dict(params)
+
+    def register_pattern(
+        self,
+        name: str,
+        keywords: list[str],
+        parameters: dict[str, Any],
+        weight: float = 1.0,
+    ) -> ContextPattern:
+        pattern = ContextPattern(
+            name=name, keywords=keywords, weight=weight, parameters=dict(parameters)
+        )
+        self._patterns.append(pattern)
+        return pattern
+
+    def classify(self, context_text: str, history: list[str] | None = None) -> ContextClassification:
+        """Classify context using weighted pattern scoring (3× current, 1× history)."""
+        text_lower = context_text.lower()
+        scores: dict[str, float] = {}
+        # Current message: 3× weight
+        for pattern in self._patterns:
+            score = 0.0
+            for kw in pattern.keywords:
+                if re.search(re.escape(kw.lower()), text_lower):
+                    score += pattern.weight
+            scores[pattern.name] = score * 3.0
+        # History: 1× weight
+        if history:
+            for h in history:
+                h_lower = h.lower()
+                for pattern in self._patterns:
+                    for kw in pattern.keywords:
+                        if re.search(re.escape(kw.lower()), h_lower):
+                            scores[pattern.name] = scores.get(pattern.name, 0.0) + pattern.weight
+        # Select matched patterns (score > 0)
+        matched = [name for name, score in scores.items() if score > 0]
+        total_score = sum(scores.values())
+        confidence = total_score / (total_score + 1.0) if total_score > 0 else 0.0
+        # Interpolate parameters
+        selected = dict(self._defaults)
+        if matched:
+            for name in matched:
+                pattern = next(p for p in self._patterns if p.name == name)
+                for k, v in pattern.parameters.items():
+                    selected[k] = v
+        return ContextClassification(
+            matched_patterns=matched,
+            confidence=min(confidence, 1.0),
+            selected_parameters=selected,
+        )
+
+    def get_patterns(self) -> list[ContextPattern]:
+        return list(self._patterns)
+
+    def count(self) -> int:
+        return len(self._patterns)

algo_cli/intelligence/context_ops.py

@@ -0,0 +1,95 @@
+"""B55. ContextOps: Token Budget Compiler + JIT References.
+
+Deterministic context selection with inclusion/exclusion reasons.
+Lazy references: don't load a file unless it fits.
+Source: ctxbudgeter pattern.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum, auto
+
+
+class InclusionReason(Enum):
+    INCLUDED = auto()
+    EXCLUDED_BUDGET = auto()
+    EXCLUDED_PRIORITY = auto()
+    EXCLUDED_DUPLICATE = auto()
+    EXCLUDED_STALE = auto()
+
+
+@dataclass
+class ContextItem:
+    name: str
+    content: str
+    priority: int = 5  # 1=highest, 10=lowest
+    token_estimate: int = 0
+    source: str = "unknown"
+    inclusion: InclusionReason = InclusionReason.INCLUDED
+    exclusion_reason: str = ""
+
+
+@dataclass
+class ContextBOM:
+    """Bill of Materials for context — auditable record of what entered/was excluded."""
+    included: list[ContextItem] = field(default_factory=list)
+    excluded: list[ContextItem] = field(default_factory=list)
+    total_tokens: int = 0
+    budget: int = 0
+
+
+class TokenBudgetCompiler:
+    """Compile context deterministically within a token budget."""
+
+    def __init__(self, budget_tokens: int = 8000):
+        self.budget = budget_tokens
+
+    def _estimate_tokens(self, content: str) -> int:
+        return max(1, len(content) // 4)
+
+    def compile(self, items: list[ContextItem]) -> ContextBOM:
+        """Select items to fit within budget, sorted by priority."""
+        # Sort by priority (1=highest), then by token estimate (smaller first)
+        sorted_items = sorted(items, key=lambda x: (x.priority, self._estimate_tokens(x.content)))
+
+        bom = ContextBOM(budget=self.budget)
+        used = 0
+
+        for item in sorted_items:
+            tokens = self._estimate_tokens(item.content)
+            item.token_estimate = tokens
+
+            if used + tokens <= self.budget:
+                item.inclusion = InclusionReason.INCLUDED
+                bom.included.append(item)
+                used += tokens
+            else:
+                item.inclusion = InclusionReason.EXCLUDED_BUDGET
+                item.exclusion_reason = f"Would exceed budget ({used + tokens}/{self.budget})"
+                bom.excluded.append(item)
+
+        bom.total_tokens = used
+        return bom
+
+    def jit_reference(self, name: str, loader: callable, current_tokens: int) -> str | None:
+        """Load a reference only if it fits in remaining budget."""
+        remaining = self.budget - current_tokens
+        if remaining <= 100:
+            return None
+        content = loader()
+        tokens = self._estimate_tokens(content)
+        if tokens <= remaining:
+            return content
+        return None  # Doesn't fit — skip
+
+    def render_bom(self, bom: ContextBOM) -> str:
+        """Render BOM as auditable markdown."""
+        lines = [f"# Context BOM ({bom.total_tokens}/{bom.budget} tokens)", ""]
+        lines.append("## Included")
+        for item in bom.included:
+            lines.append(f"- {item.name} ({item.token_estimate}t, P{item.priority})")
+        lines.append("")
+        lines.append("## Excluded")
+        for item in bom.excluded:
+            lines.append(f"- {item.name} ({item.token_estimate}t) — {item.exclusion_reason}")
+        return "\n".join(lines)

algo_cli/intelligence/count_min.py

@@ -0,0 +1,145 @@
+"""Count-Min Sketch — sublinear-space frequency estimation.
+
+A probabilistic data structure that estimates the frequency of items in a
+data stream using d independent hash functions and a d×w counter array.
+Memory is O(d*w) instead of O(unique_items).  Estimates never underestimate
+but may overestimate (false positives only).
+
+Harness use:
+  - Track tool call frequency ("how often is read_file called?")
+  - Track error frequency ("how many times has embed failed?")
+  - Detect heavy hitters in the agent loop
+  - Feed frequency data to the intuition engine
+
+Operations:
+  - update(item, count=1): O(d) — increment counters
+  - estimate(item): O(d) — return min of d counters
+  - merge(other): add two sketches
+  - heavy_hitters(threshold): items above threshold (requires tracking)
+
+Properties:
+  - Never underestimates (true_count <= estimate)
+  - Overestimate bounded by: ε·N with probability 1-δ
+    where w = ceil(e/ε), d = ceil(ln(1/δ)), N = total count
+"""
+from __future__ import annotations
+
+import hashlib
+import math
+from typing import Any
+
+
+class CountMinSketch:
+    """Count-Min Sketch with d×w counter array.
+
+    Args:
+        epsilon: Error bound (overestimate <= epsilon * total_count).
+        delta: Failure probability (Pr[overestimate > epsilon*N] < delta).
+    """
+
+    def __init__(self, epsilon: float = 0.01, delta: float = 0.01) -> None:
+        self.epsilon = epsilon
+        self.delta = delta
+        self.width = max(1, int(math.ceil(math.e / epsilon)))
+        self.depth = max(1, int(math.ceil(math.log(1 / delta))))
+        self._table: list[list[int]] = [[0] * self.width for _ in range(self.depth)]
+        self.total_count = 0
+
+    def _hash(self, item: Any, row: int) -> int:
+        """Hash item to a column index for a given row."""
+        data = f"{row}:{item}".encode("utf-8")
+        h = int.from_bytes(hashlib.md5(data).digest()[:8], "little")
+        return h % self.width
+
+    def update(self, item: Any, count: int = 1) -> None:
+        """Increment the count for item by count (default 1)."""
+        for row in range(self.depth):
+            col = self._hash(item, row)
+            self._table[row][col] += count
+        self.total_count += count
+
+    def estimate(self, item: Any) -> int:
+        """Estimate the frequency of item. Never underestimates."""
+        return min(
+            self._table[row][self._hash(item, row)]
+            for row in range(self.depth)
+        )
+
+    def merge(self, other: CountMinSketch) -> None:
+        """Merge another sketch into this one (element-wise max)."""
+        if self.width != other.width or self.depth != other.depth:
+            raise ValueError("Cannot merge sketches with different dimensions")
+        for row in range(self.depth):
+            for col in range(self.width):
+                self._table[row][col] += other._table[row][col]
+        self.total_count += other.total_count
+
+    def inner_product(self, other: CountMinSketch) -> int:
+        """Estimate the inner product of two frequency vectors.
+
+        Useful for computing dot products of item frequency distributions.
+        """
+        if self.width != other.width or self.depth != other.depth:
+            raise ValueError("Dimensions must match")
+        return min(
+            sum(self._table[row][col] * other._table[row][col]
+                for col in range(self.width))
+            for row in range(self.depth)
+        )
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            "width": self.width,
+            "depth": self.depth,
+            "memory_cells": self.width * self.depth,
+            "total_count": self.total_count,
+            "epsilon": self.epsilon,
+            "delta": self.delta,
+        }
+
+
+class HeavyHitters:
+    """Track top-K frequent items using Space-Saving algorithm.
+
+    Maintains an exact count of the top-K items seen so far, using O(K) memory.
+    Based on Metwally et al. (2005) Space-Saving algorithm.
+
+    Args:
+        k: Number of heavy hitters to track.
+    """
+
+    def __init__(self, k: int = 10) -> None:
+        self.k = k
+        self._counts: dict[str, int] = {}
+        self._total = 0
+
+    def update(self, item: Any, count: int = 1) -> None:
+        """Observe an item."""
+        key = str(item)
+        self._total += count
+        if key in self._counts:
+            self._counts[key] += count
+        elif len(self._counts) < self.k:
+            self._counts[key] = count
+        else:
+            # Replace the item with minimum count
+            min_key = min(self._counts, key=self._counts.get)
+            min_count = self._counts[min_key]
+            del self._counts[min_key]
+            self._counts[key] = min_count + count
+
+    def top_k(self) -> list[tuple[str, int]]:
+        """Return the top-K items sorted by count (descending)."""
+        return sorted(self._counts.items(), key=lambda x: -x[1])
+
+    def estimate(self, item: Any) -> int:
+        """Estimate the count of an item (lower bound)."""
+        return self._counts.get(str(item), 0)
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            "k": self.k,
+            "tracked_items": len(self._counts),
+            "total_observations": self._total,
+            "top": self.top_k()[:5],
+        }

algo_cli/intelligence/cow_state.py

@@ -0,0 +1,103 @@
+"""B71. Copy-On-Write State Isolation for Parallel Agents.
+
+COW state per parallel agent.  Shared read-only history.
+Diff merge back to main context.
+Source: gecko pattern.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class COWState:
+    """Copy-on-write state for a parallel agent."""
+    agent_id: str
+    parent_state: dict[str, Any]  # read-only reference
+    local_changes: dict[str, Any] = field(default_factory=dict)
+    deleted_keys: set[str] = field(default_factory=set)
+
+    def get(self, key: str) -> Any:
+        if key in self.deleted_keys:
+            return None
+        if key in self.local_changes:
+            return self.local_changes[key]
+        return self.parent_state.get(key)
+
+    def set(self, key: str, value: Any) -> None:
+        self.local_changes[key] = value
+        self.deleted_keys.discard(key)
+
+    def delete(self, key: str) -> None:
+        self.deleted_keys.add(key)
+        self.local_changes.pop(key, None)
+
+    def diff(self) -> dict[str, Any]:
+        """Return only the changes."""
+        return {
+            "added_or_modified": dict(self.local_changes),
+            "deleted": list(self.deleted_keys),
+        }
+
+    def snapshot(self) -> dict[str, Any]:
+        """Full snapshot: parent + local changes."""
+        result = dict(self.parent_state)
+        result.update(self.local_changes)
+        for key in self.deleted_keys:
+            result.pop(key, None)
+        return result
+
+
+class COWStateManager:
+    """Manage COW states for parallel agents."""
+
+    def __init__(self) -> None:
+        self._main_state: dict[str, Any] = {}
+        self._agent_states: dict[str, COWState] = {}
+
+    @property
+    def main_state(self) -> dict[str, Any]:
+        return dict(self._main_state)
+
+    def update_main(self, key: str, value: Any) -> None:
+        self._main_state[key] = value
+
+    def fork(self, agent_id: str) -> COWState:
+        """Create a COW fork for an agent."""
+        state = COWState(agent_id=agent_id, parent_state=self._main_state)
+        self._agent_states[agent_id] = state
+        return state
+
+    def get_state(self, agent_id: str) -> COWState | None:
+        return self._agent_states.get(agent_id)
+
+    def merge(self, agent_id: str) -> dict[str, Any]:
+        """Merge agent's changes back to main state."""
+        state = self._agent_states.get(agent_id)
+        if not state:
+            return dict(self._main_state)
+
+        # Apply changes
+        for key, value in state.local_changes.items():
+            self._main_state[key] = value
+        for key in state.deleted_keys:
+            self._main_state.pop(key, None)
+
+        # Clean up agent state
+        del self._agent_states[agent_id]
+        return dict(self._main_state)
+
+    def merge_all(self) -> dict[str, Any]:
+        """Merge all agent states back to main."""
+        for agent_id in list(self._agent_states.keys()):
+            self.merge(agent_id)
+        return dict(self._main_state)
+
+    def discard(self, agent_id: str) -> None:
+        """Discard an agent's changes without merging."""
+        self._agent_states.pop(agent_id, None)
+
+    @property
+    def active_forks(self) -> int:
+        return len(self._agent_states)

algo_cli/intelligence/critic_loop.py

@@ -0,0 +1,119 @@
+"""B64. Critic Loop + Budget Guard.
+
+Quality-gated iteration with cost/iteration kill switch.
+Source: deep-research-agent pattern.
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Any, Callable
+
+
+class CriticDimension(Enum):
+    COVERAGE = auto()
+    RECENCY = auto()
+    DEPTH = auto()
+    DIVERSITY = auto()
+    ACCURACY = auto()
+
+
+@dataclass
+class CriticScore:
+    dimension: CriticDimension
+    score: float  # 0.0 to 1.0
+    notes: str = ""
+
+
+@dataclass
+class CriticResult:
+    scores: list[CriticScore] = field(default_factory=list)
+    overall: float = 0.0
+    passed: bool = False
+    recommendations: list[str] = field(default_factory=list)
+    iteration: int = 0
+
+
+@dataclass
+class BudgetGuard:
+    max_iterations: int = 5
+    max_cost: float = 1.0  # abstract cost units
+    max_time_s: float = 120.0
+    current_cost: float = 0.0
+    current_time: float = 0.0
+    current_iterations: int = 0
+    _start_time: float = field(default_factory=time.time)
+
+    def can_proceed(self) -> bool:
+        self.current_time = time.time() - self._start_time
+        return (
+            self.current_iterations < self.max_iterations
+            and self.current_cost < self.max_cost
+            and self.current_time < self.max_time_s
+        )
+
+    def spend(self, cost: float = 0.1) -> None:
+        self.current_cost += cost
+        self.current_iterations += 1
+
+    def remaining_budget(self) -> dict[str, float]:
+        return {
+            "iterations": max(0, self.max_iterations - self.current_iterations),
+            "cost": max(0, self.max_cost - self.current_cost),
+            "time_s": max(0, self.max_time_s - (time.time() - self._start_time)),
+        }
+
+
+class CriticLoop:
+    """Quality-gated iteration loop with budget guard."""
+
+    def __init__(self, threshold: float = 0.8,
+                 budget: BudgetGuard | None = None) -> None:
+        self._threshold = threshold
+        self._budget = budget or BudgetGuard()
+        self._scorers: dict[CriticDimension, Callable[[Any], float]] = {}
+
+    def register_scorer(self, dimension: CriticDimension,
+                        scorer: Callable[[Any], float]) -> None:
+        self._scorers[dimension] = scorer
+
+    def evaluate(self, artifact: Any, iteration: int) -> CriticResult:
+        result = CriticResult(iteration=iteration)
+        for dim, scorer in self._scorers.items():
+            try:
+                score = scorer(artifact)
+                result.scores.append(CriticScore(dimension=dim, score=score))
+            except Exception:
+                result.scores.append(CriticScore(dimension=dim, score=0.0, notes="scorer error"))
+
+        result.overall = sum(s.score for s in result.scores) / len(result.scores) if result.scores else 0.0
+        result.passed = result.overall >= self._threshold
+
+        if not result.passed:
+            for s in result.scores:
+                if s.score < self._threshold:
+                    result.recommendations.append(f"Improve {s.dimension.name.lower()}: {s.score:.0%}")
+
+        return result
+
+    def run(self, produce_fn: Callable[[int], Any],
+            improve_fn: Callable[[Any, CriticResult], Any] | None = None,
+            cost_per_iteration: float = 0.1) -> tuple[Any, CriticResult]:
+        """Run the critic loop until threshold or budget exhausted."""
+        artifact = None
+        result = CriticResult()
+
+        while self._budget.can_proceed():
+            self._budget.spend(cost_per_iteration)
+            artifact = produce_fn(self._budget.current_iterations)
+            result = self.evaluate(artifact, self._budget.current_iterations)
+
+            if result.passed:
+                break
+            if improve_fn:
+                artifact = improve_fn(artifact, result)
+        else:
+            result.recommendations.append("Budget exhausted")
+
+        return artifact, result