algo-cli-runtime 0.14.0__py3-none-any.whl

algo_cli/intelligence/discovery_event_log.py

@@ -0,0 +1,92 @@
+"""H4 — Harness Discovery Event Log.
+
+Structured events for the full discovery lifecycle.
+Mined from T3MP3ST EventEmitter (finding:discovered, finding:validated, finding:retracted).
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+
+class EventType:
+    DISCOVERED = "discovered"
+    VALIDATED = "validated"
+    RETRACTED = "retracted"
+    PROPOSED = "proposed"
+    ENRICHED = "enriched"
+    PROPAGATED = "propagated"
+
+
+@dataclass
+class DiscoveryEvent:
+    """A single lifecycle event."""
+
+    event_type: str
+    target_id: str
+    timestamp: float = field(default_factory=time.time)
+    source: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "event_type": self.event_type,
+            "target_id": self.target_id,
+            "timestamp": self.timestamp,
+            "source": self.source,
+            "metadata": dict(self.metadata),
+        }
+
+
+class DiscoveryEventLog:
+    """Append-only event log with listener support."""
+
+    def __init__(self, max_events: int = 10_000) -> None:
+        self._events: list[DiscoveryEvent] = []
+        self._listeners: list[Callable[[DiscoveryEvent], None]] = []
+        self._max_events = max_events
+
+    def emit(
+        self,
+        event_type: str,
+        target_id: str,
+        source: str = "",
+        metadata: dict[str, Any] | None = None,
+    ) -> DiscoveryEvent:
+        event = DiscoveryEvent(
+            event_type=event_type,
+            target_id=target_id,
+            source=source,
+            metadata=metadata or {},
+        )
+        self._events.append(event)
+        if len(self._events) > self._max_events:
+            self._events = self._events[-self._max_events:]
+        for listener in self._listeners:
+            listener(event)
+        return event
+
+    def on(self, listener: Callable[[DiscoveryEvent], None]) -> None:
+        self._listeners.append(listener)
+
+    def query(
+        self,
+        event_type: str | None = None,
+        target_id: str | None = None,
+    ) -> list[DiscoveryEvent]:
+        results = self._events
+        if event_type is not None:
+            results = [e for e in results if e.event_type == event_type]
+        if target_id is not None:
+            results = [e for e in results if e.target_id == target_id]
+        return list(results)
+
+    def all(self) -> list[DiscoveryEvent]:
+        return list(self._events)
+
+    def count(self) -> int:
+        return len(self._events)
+
+    def clear_listeners(self) -> None:
+        self._listeners.clear()

algo_cli/intelligence/document_ingest.py

@@ -0,0 +1,298 @@
+"""B34. Converter Pipeline + Markdown Normalization (MarkItDown Pattern).
+
+Converts arbitrary file formats into token-efficient Markdown before RAG or
+agent analysis.  Uses a registry of format-specific converters with narrow
+readers (local file, stream, URL) and security boundaries.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+import re
+import zipfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterable
+from xml.etree import ElementTree
+
+# ── data types ────────────────────────────────────────────────────────
+
+
+@dataclass
+class ConversionResult:
+    """Outcome of a single conversion."""
+
+    markdown: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    warnings: list[str] = field(default_factory=list)
+
+
+class UnsupportedFormat(Exception):
+    """No registered converter can handle the given file."""
+
+
+# ── converter base ────────────────────────────────────────────────────
+
+
+class Converter:
+    """Base converter — override ``supports`` and ``convert_local``."""
+
+    def supports(self, path: Path) -> bool:
+        raise NotImplementedError
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        raise NotImplementedError
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        raise NotImplementedError
+
+
+# ── built-in converters ───────────────────────────────────────────────
+
+
+class CSVConverter(Converter):
+    """CSV → Markdown table."""
+
+    def supports(self, path: Path) -> bool:
+        return path.suffix.lower() in {".csv", ".tsv"}
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        delim = "\t" if path.suffix.lower() == ".tsv" else ","
+        with open(path, encoding="utf-8", newline="") as fh:
+            return self._from_reader(csv.reader(fh, delimiter=delim), path)
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        text = data.decode("utf-8", errors="replace")
+        return self._from_reader(csv.reader(io.StringIO(text)), Path(name))
+
+    @staticmethod
+    def _from_reader(rows: Iterable[list[str]], path: Path) -> ConversionResult:
+        lines = list(rows)
+        if not lines:
+            return ConversionResult(markdown="", metadata={"source": str(path)})
+        header = lines[0]
+        md = "| " + " | ".join(header) + " |\n"
+        md += "|" + "---|" * len(header) + "\n"
+        for row in lines[1:]:
+            # pad / truncate to header length
+            cells = list(row) + [""] * (len(header) - len(row))
+            md += "| " + " | ".join(cells[: len(header)]) + " |\n"
+        return ConversionResult(
+            markdown=md,
+            metadata={"source": str(path), "rows": len(lines) - 1, "columns": len(header)},
+        )
+
+
+class JSONConverter(Converter):
+    """JSON → fenced code block."""
+
+    def supports(self, path: Path) -> bool:
+        return path.suffix.lower() == ".json"
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        with open(path, encoding="utf-8") as fh:
+            data = json.load(fh)
+        return self._from_data(data, path)
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        return self._from_data(json.loads(data), Path(name))
+
+    @staticmethod
+    def _from_data(data: Any, path: Path) -> ConversionResult:
+        md = f"```json\n{json.dumps(data, indent=2)}\n```\n"
+        return ConversionResult(
+            markdown=md,
+            metadata={"source": str(path), "type": type(data).__name__},
+        )
+
+
+class XMLConverter(Converter):
+    """XML → indented text inside a code fence."""
+
+    def supports(self, path: Path) -> bool:
+        return path.suffix.lower() in {".xml", ".svg", ".xsd", ".xsl"}
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        tree = ElementTree.parse(path)
+        return self._from_tree(tree, path)
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        return self._from_tree(ElementTree.fromstring(data), Path(name))
+
+    @staticmethod
+    def _from_tree(tree: Any, path: Path) -> ConversionResult:
+        if isinstance(tree, ElementTree.ElementTree):
+            root = tree.getroot()
+        else:
+            root = tree
+        text = ElementTree.tostring(root, encoding="unicode")
+        md = f"```xml\n{text}\n```\n"
+        return ConversionResult(
+            markdown=md,
+            metadata={"source": str(path), "root_tag": root.tag},
+        )
+
+
+class HTMLConverter(Converter):
+    """Strip HTML tags → plain text with basic structure preservation."""
+
+    _TAG_RE = re.compile(r"<[^>]+>")
+    _BLOCK_RE = re.compile(r"</(p|div|h[1-6]|li|tr|br)\s*>", re.I)
+
+    def supports(self, path: Path) -> bool:
+        return path.suffix.lower() in {".html", ".htm"}
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        html = path.read_text(encoding="utf-8", errors="replace")
+        return self._from_html(html, path)
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        return self._from_html(data.decode("utf-8", errors="replace"), Path(name))
+
+    def _from_html(self, html: str, path: Path) -> ConversionResult:
+        # insert newlines after block-level closes
+        text = self._BLOCK_RE.sub("\n", html)
+        text = self._TAG_RE.sub("", text)
+        # collapse excessive blank lines
+        text = re.sub(r"\n{3,}", "\n\n", text).strip()
+        md = f"{text}\n"
+        return ConversionResult(
+            markdown=md,
+            metadata={"source": str(path), "chars": len(text)},
+        )
+
+
+class TextConverter(Converter):
+    """Plain text / Markdown passthrough."""
+
+    def supports(self, path: Path) -> bool:
+        return path.suffix.lower() in {".txt", ".md", ".markdown", ".rst", ".log"}
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        text = path.read_text(encoding="utf-8", errors="replace")
+        return ConversionResult(
+            markdown=text,
+            metadata={"source": str(path), "chars": len(text)},
+        )
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        text = data.decode("utf-8", errors="replace")
+        return ConversionResult(
+            markdown=text,
+            metadata={"source": name, "chars": len(text)},
+        )
+
+
+class ZipConverter(Converter):
+    """Traverse ZIP archives and convert each entry with a sub-registry."""
+
+    def __init__(self, sub_registry: "ConverterRegistry", max_files: int = 50, max_bytes: int = 5_000_000):
+        self.sub_registry = sub_registry
+        self.max_files = max_files
+        self.max_bytes = max_bytes
+
+    def supports(self, path: Path) -> bool:
+        return path.suffix.lower() == ".zip"
+
+    def convert_local(self, path: Path) -> ConversionResult:
+        parts: list[str] = []
+        warnings: list[str] = []
+        total_bytes = 0
+        file_count = 0
+        with zipfile.ZipFile(path) as zf:
+            for info in zf.infolist():
+                if info.is_dir():
+                    continue
+                if file_count >= self.max_files:
+                    warnings.append(f"max_files={self.max_files} reached, remaining entries skipped")
+                    break
+                if total_bytes + info.file_size > self.max_bytes:
+                    warnings.append(f"max_bytes={self.max_bytes} reached, remaining entries skipped")
+                    break
+                data = zf.read(info)
+                total_bytes += len(data)
+                file_count += 1
+                inner_path = Path(info.filename)
+                try:
+                    sub = self.sub_registry.convert_local(inner_path, _data_override=data)
+                except UnsupportedFormat:
+                    parts.append(f"## {info.filename}\n\n*(binary or unsupported)*\n")
+                    continue
+                parts.append(f"## {info.filename}\n\n{sub.markdown}")
+        return ConversionResult(
+            markdown="\n\n".join(parts),
+            metadata={"source": str(path), "entries": file_count, "total_bytes": total_bytes},
+            warnings=warnings,
+        )
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        # write to temp, delegate
+        import tempfile
+        with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tf:
+            tf.write(data)
+            tf_path = Path(tf.name)
+        try:
+            return self.convert_local(tf_path)
+        finally:
+            os.unlink(tf_path)
+
+
+# ── registry ──────────────────────────────────────────────────────────
+
+
+class ConverterRegistry:
+    """Ordered registry of converters with narrow reader methods."""
+
+    def __init__(self) -> None:
+        self._converters: list[Converter] = []
+
+    def register(self, converter: Converter) -> None:
+        self._converters.append(converter)
+
+    def _find(self, path: Path) -> Converter:
+        for c in self._converters:
+            if c.supports(path):
+                return c
+        raise UnsupportedFormat(f"No converter for {path.suffix}")
+
+    def convert_local(self, path: Path, _data_override: bytes | None = None) -> ConversionResult:
+        converter = self._find(path)
+        if _data_override is not None and hasattr(converter, "convert_stream"):
+            return converter.convert_stream(_data_override, str(path))
+        return converter.convert_local(path)
+
+    def convert_stream(self, data: bytes, name: str = "") -> ConversionResult:
+        path = Path(name or "stream")
+        converter = self._find(path)
+        return converter.convert_stream(data, name)
+
+    def supported_suffixes(self) -> set[str]:
+        suffixes: set[str] = set()
+        for c in self._converters:
+            # probe common suffixes
+            for suf in [".csv", ".tsv", ".json", ".xml", ".html", ".htm", ".txt", ".md", ".zip"]:
+                if c.supports(Path(f"file{suf}")):
+                    suffixes.add(suf)
+        return suffixes
+
+
+def default_registry() -> ConverterRegistry:
+    """Build a registry with all built-in converters."""
+    reg = ConverterRegistry()
+    reg.register(CSVConverter())
+    reg.register(JSONConverter())
+    reg.register(XMLConverter())
+    reg.register(HTMLConverter())
+    reg.register(TextConverter())
+    # zip needs a sub-registry without itself to avoid recursion
+    sub = ConverterRegistry()
+    sub.register(CSVConverter())
+    sub.register(JSONConverter())
+    sub.register(XMLConverter())
+    sub.register(HTMLConverter())
+    sub.register(TextConverter())
+    reg.register(ZipConverter(sub))
+    return reg

algo_cli/intelligence/dual_layer_validate.py

@@ -0,0 +1,151 @@
+"""H18 — Dual-Layer Validation.
+
+Validator probes check claims, then an independent skeptic attempts to refute.
+Mined from GLOSSOPETRAE VALIDATION.md: six validators + independent skeptic.
+
+LLM integration: optionally uses an LLM as the skeptic. Falls back to
+rule-based refutation when no model is available.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+
+@dataclass
+class ValidationResult:
+    """Result of a single validator probe."""
+
+    validator_name: str
+    passed: bool
+    message: str = ""
+    evidence: list[str] = field(default_factory=list)
+
+
+@dataclass
+class DualLayerResult:
+    """Combined result of validator + skeptic layers."""
+
+    entry_id: str
+    validator_results: list[ValidationResult] = field(default_factory=list)
+    skeptic_result: ValidationResult | None = None
+    final_verdict: bool = False
+    confidence: float = 0.0
+
+
+def run_validators(
+    entry_id: str,
+    claims: dict[str, Any],
+    validators: list[tuple[str, Callable[[str, dict], ValidationResult]]],
+) -> list[ValidationResult]:
+    """Run multiple validator probes on a claim set.
+
+    Args:
+        entry_id: The entry being validated.
+        claims: The claims to validate.
+        validators: List of (name, validator_fn) tuples.
+
+    Returns:
+        List of ValidationResult from each validator.
+    """
+    results: list[ValidationResult] = []
+    for name, fn in validators:
+        result = fn(entry_id, claims)
+        if not isinstance(result, ValidationResult):
+            result = ValidationResult(
+                validator_name=name,
+                passed=bool(result),
+            )
+        results.append(result)
+    return results
+
+
+def run_skeptic(
+    entry_id: str,
+    claims: dict[str, Any],
+    validator_results: list[ValidationResult],
+    model_client: Any | None = None,
+) -> ValidationResult:
+    """Run an independent skeptic that attempts to refute the validators.
+
+    Args:
+        entry_id: The entry being scrutinized.
+        claims: The claims to attack.
+        validator_results: Results from the validator layer.
+        model_client: Optional LLM for deeper refutation.
+
+    Returns:
+        ValidationResult from the skeptic's perspective.
+    """
+    # Rule-based skeptic: find any validator that failed
+    failed_validators = [r for r in validator_results if not r.passed]
+    if failed_validators:
+        return ValidationResult(
+            validator_name="skeptic",
+            passed=False,
+            message=f"Skeptic confirms failures from: {[r.validator_name for r in failed_validators]}",
+            evidence=[r.message for r in failed_validators],
+        )
+
+    # Rule-based skeptic: check for unsupported claims
+    unsupported = []
+    for key, value in claims.items():
+        if value is None:
+            unsupported.append(f"{key} is None")
+        elif isinstance(value, str) and not value.strip():
+            unsupported.append(f"{key} is empty")
+
+    if unsupported:
+        return ValidationResult(
+            validator_name="skeptic",
+            passed=False,
+            message="Skeptic found unsupported claims",
+            evidence=unsupported,
+        )
+
+    return ValidationResult(
+        validator_name="skeptic",
+        passed=True,
+        message="Skeptic could not refute validators",
+    )
+
+
+def dual_layer_validate(
+    entry_id: str,
+    claims: dict[str, Any],
+    validators: list[tuple[str, Callable[[str, dict], ValidationResult]]],
+    model_client: Any | None = None,
+) -> DualLayerResult:
+    """Run full dual-layer validation.
+
+    Args:
+        entry_id: The entry being validated.
+        claims: The claims to validate.
+        validators: List of (name, validator_fn) tuples.
+        model_client: Optional LLM for skeptic.
+
+    Returns:
+        DualLayerResult with both layers and final verdict.
+    """
+    validator_results = run_validators(entry_id, claims, validators)
+    skeptic_result = run_skeptic(entry_id, claims, validator_results, model_client)
+
+    # Final verdict: all validators must pass AND skeptic must pass
+    all_validators_pass = all(r.passed for r in validator_results)
+    final_verdict = all_validators_pass and skeptic_result.passed
+
+    # Confidence: fraction of validators that passed
+    if validator_results:
+        validator_pass_rate = sum(1 for r in validator_results if r.passed) / len(validator_results)
+    else:
+        validator_pass_rate = 0.0
+
+    confidence = validator_pass_rate * (1.0 if skeptic_result.passed else 0.5)
+
+    return DualLayerResult(
+        entry_id=entry_id,
+        validator_results=validator_results,
+        skeptic_result=skeptic_result,
+        final_verdict=final_verdict,
+        confidence=confidence,
+    )

algo_cli/intelligence/echo_fidelity.py

@@ -0,0 +1,73 @@
+"""H7 — Echo-Fidelity Guard.
+
+Distinguishes ``None`` (unmeasured) from ``0.0`` (measured failure) from a
+valid float.  Prevents fabrications where a missing measurement is silently
+treated as a zero (or vice-versa).
+
+Source: GLOSSOPETRAE experiments/lib — null vs 0 distinction.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class GuardResult:
+    """Outcome of an echo-fidelity check."""
+
+    value: Optional[float]
+    status: str  # "unmeasured" | "measured" | "invalid"
+    message: str
+
+    @property
+    def is_measured(self) -> bool:
+        return self.status == "measured"
+
+
+class EchoFidelityGuard:
+    """Guard that distinguishes unmeasured from measured-zero values."""
+
+    def check(self, value: object) -> GuardResult:
+        """Check a value and return its fidelity status.
+
+        - ``None`` → unmeasured (no data collected)
+        - ``float`` or ``int`` in valid range → measured
+        - ``float`` or ``int`` outside [lo, hi] → invalid
+        """
+        if value is None:
+            return GuardResult(
+                value=None,
+                status="unmeasured",
+                message="No measurement was taken (value is None).",
+            )
+        if isinstance(value, bool):
+            return GuardResult(
+                value=None,
+                status="invalid",
+                message="Boolean is not a valid measurement.",
+            )
+        if isinstance(value, (int, float)):
+            v = float(value)
+            return GuardResult(
+                value=v,
+                status="measured",
+                message=f"Measurement recorded: {v}",
+            )
+        return GuardResult(
+            value=None,
+            status="invalid",
+            message=f"Expected numeric or None, got {type(value).__name__}.",
+        )
+
+    def check_many(self, values: list[object]) -> list[GuardResult]:
+        """Check a batch of values."""
+        return [self.check(v) for v in values]
+
+    def measured_values(self, values: list[object]) -> list[float]:
+        """Return only the measured float values, skipping unmeasured and invalid."""
+        return [
+            r.value
+            for r in self.check_many(values)
+            if r.is_measured and r.value is not None
+        ]