errorsense 0.1.0__py3-none-any.whl

errorsense/models.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class SenseResult:
+    """Result from classification — produced by rulesets or skills."""
+
+    label: str
+    confidence: float
+    phase: str = ""
+    skill_name: str = ""
+    reason: str | None = None  # only set when explain=True, LLM phases only
+
+
+@dataclass(frozen=True)
+class TrailResult:
+    """Result from trail() — classification + threshold state.
+
+    If a review ran (threshold hit + review enabled), label and reason
+    reflect the review's verdict. If the review changed the label,
+    the history entry is updated and counts are adjusted.
+    """
+
+    label: str
+    confidence: float
+    phase: str
+    skill_name: str
+    at_threshold: bool
+    reason: str | None = None  # LLM review explanation, None if no review ran
+
+
+@dataclass(frozen=True)
+class TrailingConfig:
+    """Configuration for trailing (stateful error tracking).
+
+    Args:
+        threshold: Number of counted errors before review triggers.
+        count_labels: Only these labels count toward threshold.
+        history_size: Max errors kept per key (ring buffer).
+        review: Whether to LLM-review history when threshold hit.
+            None = auto (True if LLM phase exists, False if not).
+            True = force (raises if no LLM phase).
+            False = never.
+    """
+
+    threshold: int = 3
+    count_labels: list[str] | None = None
+    history_size: int = 10
+    review: bool | None = None

errorsense/phase.py

@@ -0,0 +1,192 @@
+"""Phase — named stage in the classification pipeline."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import replace
+from typing import Any
+
+from errorsense.llm import LLMClient, LLMConfig
+from errorsense.models import SenseResult
+from errorsense.ruleset import Ruleset
+from errorsense.signal import Signal
+from errorsense.skill import Skill
+
+logger = logging.getLogger("errorsense")
+
+__all__ = ["Phase"]
+
+
+class Phase:
+    """A named stage in the classification pipeline.
+
+    Each phase contains either rulesets (deterministic) or skills (LLM).
+    Not both.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        rulesets: list[Ruleset] | None = None,
+        skills: list[Skill] | None = None,
+        llm: LLMConfig | None = None,
+    ) -> None:
+        if not name:
+            raise ValueError("Phase requires a non-empty name")
+
+        has_rulesets = rulesets is not None and len(rulesets) > 0
+        has_skills = skills is not None and len(skills) > 0
+
+        if has_rulesets and has_skills:
+            raise ValueError(
+                f"Phase {name!r}: cannot mix rulesets and skills. "
+                "Use rulesets OR (skills + llm), not both."
+            )
+        if not has_rulesets and not has_skills:
+            raise ValueError(
+                f"Phase {name!r}: must have at least one ruleset or skill."
+            )
+        if has_skills and not llm:
+            raise ValueError(
+                f"Phase {name!r}: skills require llm=LLMConfig(...)."
+            )
+        if has_rulesets and llm:
+            logger.warning(
+                "Phase %r: llm config ignored for ruleset phase.", name
+            )
+
+        self.name = name
+        self.rulesets = rulesets or []
+        self.skills = skills or []
+        self.llm = llm
+        self.is_llm_phase = has_skills
+        self._categories: list[str] = []
+        self._llm_client: LLMClient | None = None
+
+        if self.is_llm_phase and llm:
+            self._llm_client = LLMClient(llm)
+
+    def set_categories(self, categories: list[str]) -> None:
+        self._categories = list(categories)
+
+    def classify(self, signal: Signal, explain: bool = False) -> SenseResult | None:
+        """Sync classification. Full pipeline — rulesets or LLM."""
+        if self.is_llm_phase:
+            return self._run_skills_sync(signal, explain)
+        return self._run_rulesets(signal)
+
+    async def async_classify(self, signal: Signal, explain: bool = False) -> SenseResult | None:
+        """Async classification. Full pipeline — rulesets or LLM."""
+        if self.is_llm_phase:
+            return await self._run_skills_async(signal, explain)
+        return self._run_rulesets(signal)
+
+    def _run_rulesets(self, signal: Signal) -> SenseResult | None:
+        for ruleset in self.rulesets:
+            try:
+                result = ruleset.classify(signal)
+            except Exception as e:
+                logger.warning(
+                    "Phase %r: ruleset %s raised %s: %s",
+                    self.name, type(ruleset).__name__, type(e).__name__, e,
+                )
+                continue
+            if result is not None:
+                return self._stamp_phase(result, type(ruleset).__name__)
+        return None
+
+    def _run_skills_sync(self, signal: Signal, explain: bool) -> SenseResult | None:
+        if not self._llm_client:
+            return None
+
+        best: SenseResult | None = None
+        for skill in self.skills:
+            try:
+                r = self._run_one_skill_sync(signal, skill, explain)
+            except Exception as e:
+                logger.warning("Phase %r: skill %r failed: %s", self.name, skill.name, e)
+                continue
+            if r is None:
+                continue
+            result = self._stamp_phase(r, r.skill_name)
+            if best is None or result.confidence > best.confidence:
+                best = result
+        return best
+
+    async def _run_skills_async(self, signal: Signal, explain: bool) -> SenseResult | None:
+        if not self._llm_client:
+            return None
+
+        results = await asyncio.gather(
+            *[self._run_one_skill_async(signal, skill, explain) for skill in self.skills],
+            return_exceptions=True,
+        )
+
+        best: SenseResult | None = None
+        for r in results:
+            if isinstance(r, Exception):
+                logger.warning("Phase %r: skill failed: %s", self.name, r)
+                continue
+            if r is None:
+                continue
+            result = self._stamp_phase(r, r.skill_name)
+            if best is None or result.confidence > best.confidence:
+                best = result
+        return best
+
+    def _run_one_skill_sync(self, signal: Signal, skill: Skill, explain: bool) -> SenseResult | None:
+        if skill.llm is not None:
+            client = LLMClient(skill.llm)
+            try:
+                return client.classify_sync(signal, skill, self._categories, include_reason=explain)
+            finally:
+                client.close_sync()
+        return self._llm_client.classify_sync(signal, skill, self._categories, include_reason=explain)
+
+    async def _run_one_skill_async(self, signal: Signal, skill: Skill, explain: bool) -> SenseResult | None:
+        if skill.llm is not None:
+            client = LLMClient(skill.llm)
+            try:
+                return await client.classify_async(signal, skill, self._categories, include_reason=explain)
+            finally:
+                await client.close_async()
+        return await self._llm_client.classify_async(signal, skill, self._categories, include_reason=explain)
+
+    def run_llm_call(
+        self, signal: Signal, skill: Skill, categories: list[str],
+    ) -> SenseResult | None:
+        """Run a single sync LLM call. Public API for Tracker reclassification."""
+        if not self._llm_client:
+            return None
+        return self._llm_client.classify_sync(signal, skill, categories, include_reason=True)
+
+    async def async_run_llm_call(
+        self, signal: Signal, skill: Skill, categories: list[str],
+    ) -> SenseResult | None:
+        """Run a single async LLM call. Public API for Tracker reclassification."""
+        if not self._llm_client:
+            return None
+        return await self._llm_client.classify_async(signal, skill, categories, include_reason=True)
+
+    def _stamp_phase(self, result: SenseResult, skill_name: str) -> SenseResult:
+        updates: dict[str, Any] = {}
+        if not result.phase:
+            updates["phase"] = self.name
+        if not result.skill_name:
+            updates["skill_name"] = skill_name
+        if updates:
+            return replace(result, **updates)
+        return result
+
+    def close_sync(self) -> None:
+        if self._llm_client:
+            self._llm_client.close_sync()
+
+    async def close_async(self) -> None:
+        if self._llm_client:
+            await self._llm_client.close_async()
+
+    async def close(self) -> None:
+        if self._llm_client:
+            await self._llm_client.close()

errorsense/presets/__init__.py

@@ -0,0 +1,5 @@
+"""Built-in presets — opinionated pre-configured ErrorSense instances."""
+
+from errorsense.presets.http_gateway import http, http_no_llm
+
+__all__ = ["http", "http_no_llm"]

errorsense/presets/http_gateway.py

@@ -0,0 +1,72 @@
+"""HTTP presets — client vs server error classification."""
+
+from __future__ import annotations
+
+from errorsense.engine import ErrorSense
+from errorsense.llm import LLMConfig
+from errorsense.phase import Phase
+from errorsense.ruleset import Ruleset
+from errorsense.skill import Skill
+
+__all__ = ["http", "http_no_llm"]
+
+
+def _ruleset_phases(extra_rulesets: list[Ruleset] | None = None) -> list[Phase]:
+    """Shared ruleset phases for both http() and http_no_llm()."""
+    return [
+        Phase("rules", rulesets=[
+            Ruleset(field="status_code", match={
+                "4xx": "client", 502: "server", 503: "server", 504: "server",
+            }),
+            Ruleset(field="headers.content-type", match={
+                "text/html": "server", "application/json": None,
+            }),
+        ]),
+        Phase("patterns", rulesets=[
+            Ruleset(field="body", patterns=[
+                ("server", [r"Bad Gateway", r"Service Unavailable", r"Gateway Timeout"]),
+            ]),
+            *(extra_rulesets or []),
+        ]),
+    ]
+
+
+def http(
+    llm: LLMConfig,
+    extra_rulesets: list[Ruleset] | None = None,
+) -> ErrorSense:
+    """HTTP error classification with LLM: client, server, or undecided.
+
+    Rulesets handle clear-cut cases (4xx, 502/503/504, HTML responses).
+    LLM handles ambiguous errors — this is where ErrorSense earns its keep.
+
+    Args:
+        llm: LLM connection config (required).
+        extra_rulesets: Additional rulesets appended to the patterns phase.
+    """
+    phases = _ruleset_phases(extra_rulesets)
+    phases.append(Phase("llm", skills=[Skill("http_classifier")], llm=llm))
+
+    return ErrorSense(
+        categories=["client", "server", "undecided"],
+        pipeline=phases,
+        default="undecided",
+    )
+
+
+def http_no_llm(
+    extra_rulesets: list[Ruleset] | None = None,
+) -> ErrorSense:
+    """HTTP error classification without LLM: client, server, or undecided.
+
+    Only classifies clear-cut cases (status codes, gateway patterns).
+    Ambiguous errors are "undecided".
+
+    Args:
+        extra_rulesets: Additional rulesets appended to the patterns phase.
+    """
+    return ErrorSense(
+        categories=["client", "server", "undecided"],
+        pipeline=_ruleset_phases(extra_rulesets),
+        default="undecided",
+    )

errorsense/ruleset.py

@@ -0,0 +1,165 @@
+"""Ruleset — deterministic (non-LLM) classification logic."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+from errorsense.models import SenseResult
+from errorsense.signal import Signal
+
+logger = logging.getLogger("errorsense")
+
+__all__ = ["Ruleset"]
+
+
+def _resolve_dotted(data: Any, path: str) -> Any:
+    """Resolve a dotted path like 'error.type' into nested dict access."""
+    current = data
+    for part in path.split("."):
+        if isinstance(current, dict):
+            current = current.get(part)
+        else:
+            return None
+    return current
+
+
+class Ruleset:
+    """Deterministic classification logic.
+
+    Each ruleset does one thing: either field matching (match=) or regex
+    patterns (patterns=). Not both. Subclass and override classify() for
+    custom logic beyond config.
+    """
+
+    def __init__(
+        self,
+        field: str | None = None,
+        match: dict[Any, str | None] | None = None,
+        patterns: list[tuple[str, list[str]]] | None = None,
+        case_sensitive: bool = False,
+    ) -> None:
+        if type(self) is Ruleset:
+            if not field:
+                raise ValueError("Ruleset requires a 'field' parameter")
+            if match is not None and patterns is not None:
+                raise ValueError(
+                    "Ruleset takes match= OR patterns=, not both. "
+                    "Use separate rulesets in the same phase."
+                )
+            if match is None and patterns is None:
+                raise ValueError("Ruleset requires either match= or patterns=")
+
+        self._init_fields(field, match, patterns, case_sensitive)
+
+    def _init_fields(
+        self,
+        field: str | None,
+        match: dict[Any, str | None] | None,
+        patterns: list[tuple[str, list[str]]] | None,
+        case_sensitive: bool,
+    ) -> None:
+        self.field = field
+        self._match = match
+        self._range_keys: dict[str, str] = {}
+        self._exact_keys: dict[Any, str | None] = {}
+        self._compiled: list[tuple[str, list[re.Pattern[str]]]] | None = None
+
+        if match:
+            self._split_match_keys(match)
+        if patterns:
+            flags = 0 if case_sensitive else re.IGNORECASE
+            self._compiled = [
+                (label, [re.compile(p, flags) for p in pats])
+                for label, pats in patterns
+            ]
+
+    def _split_match_keys(self, match: dict[Any, str | None]) -> None:
+        for key, value in match.items():
+            if isinstance(key, str) and len(key) == 3 and key[0].isdigit() and key.endswith("xx"):
+                if value is not None:
+                    self._range_keys[key] = value
+            else:
+                self._exact_keys[key] = value
+
+    def referenced_labels(self) -> set[str]:
+        """Return set of label strings this ruleset can produce. Used by engine validation."""
+        match = getattr(self, "_match", None)
+        if match is None:
+            return set()
+        return {v for v in match.values() if isinstance(v, str)}
+
+    def classify(self, signal: Signal) -> SenseResult | None:
+        """Classify a signal. Override in subclass for custom logic."""
+        value = self._resolve_field(signal)
+        if value is None:
+            return None
+
+        if self._match is not None:
+            return self._match_value(value)
+        if self._compiled is not None:
+            return self._match_patterns(value)
+        return None
+
+    def _resolve_field(self, signal: Signal) -> Any:
+        field = self.field
+        if field is None:
+            return None
+
+        if field.startswith("headers."):
+            headers = signal.get("headers")
+            if not hasattr(headers, "get"):
+                return None
+            header_name = field[len("headers."):]
+            return headers.get(header_name, "")
+
+        if field.startswith("body."):
+            body = signal.get("body")
+            if not isinstance(body, str):
+                return None
+            try:
+                parsed = json.loads(body)
+            except (json.JSONDecodeError, TypeError):
+                logger.debug("Ruleset %r: failed to parse JSON body", field)
+                return None
+            if not isinstance(parsed, dict):
+                return None
+            dot_path = field[len("body."):]
+            return _resolve_dotted(parsed, dot_path)
+
+        return signal.get(field)
+
+    def _match_value(self, value: Any) -> SenseResult | None:
+        field = self.field
+
+        if value in self._exact_keys:
+            label = self._exact_keys[value]
+            if label is None:
+                return None
+            return SenseResult(label=label, confidence=1.0)
+
+        if isinstance(value, int) and self._range_keys:
+            range_key = f"{value // 100}xx"
+            if range_key in self._range_keys:
+                label = self._range_keys[range_key]
+                return SenseResult(label=label, confidence=1.0)
+
+        if isinstance(value, str) and self._exact_keys:
+            for pattern, label in self._exact_keys.items():
+                if isinstance(pattern, str) and pattern in value:
+                    if label is None:
+                        return None
+                    return SenseResult(label=label, confidence=1.0)
+
+        return None
+
+    def _match_patterns(self, value: Any) -> SenseResult | None:
+        if not isinstance(value, str):
+            return None
+        for label, compiled_pats in self._compiled:
+            for pat in compiled_pats:
+                if pat.search(value):
+                    return SenseResult(label=label, confidence=0.9)
+        return None

errorsense/signal.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import traceback
+from types import MappingProxyType
+from typing import Any
+
+
+def _deep_freeze(obj: Any) -> Any:
+    """Recursively freeze dicts into MappingProxyType and lists into tuples."""
+    if isinstance(obj, dict):
+        return MappingProxyType({k: _deep_freeze(v) for k, v in obj.items()})
+    if isinstance(obj, list):
+        return tuple(_deep_freeze(item) for item in obj)
+    return obj
+
+
+class Signal:
+    """Immutable container for error/event data.
+
+    All values are deep-frozen at construction time — skills get a
+    truly read-only view. Dict-like access for convenience.
+    """
+
+    __slots__ = ("_data",)
+
+    def __init__(self, data: dict[str, Any] | None = None, **kwargs: Any) -> None:
+        raw = {**(data or {}), **kwargs}
+        object.__setattr__(self, "_data", _deep_freeze(raw))
+
+    def __getitem__(self, key: str) -> Any:
+        return self._data[key]
+
+    def get(self, key: str, default: Any = None) -> Any:
+        return self._data.get(key, default)
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._data
+
+    def keys(self) -> Any:
+        return self._data.keys()
+
+    def values(self) -> Any:
+        return self._data.values()
+
+    def items(self) -> Any:
+        return self._data.items()
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a mutable deep copy of the signal data."""
+        return _thaw(self._data)
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        raise AttributeError("Signal is immutable")
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        raise TypeError("Signal is immutable")
+
+    def __delitem__(self, key: str) -> None:
+        raise TypeError("Signal is immutable")
+
+    def __repr__(self) -> str:
+        return f"Signal({dict(self._data)!r})"
+
+    @classmethod
+    def from_http(
+        cls,
+        status_code: int,
+        body: str = "",
+        headers: dict[str, str] | None = None,
+    ) -> Signal:
+        return cls(
+            {
+                "status_code": status_code,
+                "body": body,
+                "headers": headers or {},
+            }
+        )
+
+    @classmethod
+    def from_grpc(cls, code: int, details: str = "") -> Signal:
+        return cls({"grpc_code": code, "details": details})
+
+    @classmethod
+    def from_exception(cls, exc: BaseException) -> Signal:
+        return cls(
+            {
+                "exception_type": type(exc).__name__,
+                "message": str(exc),
+                "traceback": traceback.format_exception(type(exc), exc, exc.__traceback__),
+            }
+        )
+
+
+def _thaw(obj: Any) -> Any:
+    """Recursively convert MappingProxyType back to dict and tuples to lists."""
+    if isinstance(obj, (MappingProxyType, dict)):
+        return {k: _thaw(v) for k, v in obj.items()}
+    if isinstance(obj, (tuple, list)):
+        return [_thaw(item) for item in obj]
+    return obj

errorsense/skill.py

@@ -0,0 +1,70 @@
+"""Skill — LLM domain instructions loaded from markdown files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from errorsense.llm import LLMConfig
+
+__all__ = ["Skill"]
+
+_BUILT_IN_SKILLS_DIR = Path(__file__).parent / "skills"
+
+
+class Skill:
+    """Domain-specific instructions for LLM classification.
+
+    Instructions are loaded from a markdown file by default. Built-in skills
+    live in errorsense/skills/. Custom skills can point to any file path.
+
+    For programmatic use (e.g. Tracker reclassification), inline instructions=
+    is also supported.
+
+    Args:
+        name: Skill name. If no path or instructions given, looks for {name}.md
+              in the built-in skills directory.
+        path: Explicit path to a .md file. Overrides built-in lookup.
+        instructions: Inline instructions string. Overrides file loading.
+        prompt_template: Override the default LLM prompt template.
+        temperature: LLM temperature (default: 0.0 for determinism).
+        llm: Per-skill LLMConfig override.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        path: str | Path | None = None,
+        instructions: str | None = None,
+        prompt_template: str | None = None,
+        temperature: float = 0.0,
+        llm: LLMConfig | None = None,
+    ) -> None:
+        if not name:
+            raise ValueError("Skill requires a non-empty 'name'")
+
+        self.name = name
+        self.prompt_template = prompt_template
+        self.temperature = temperature
+        self.llm = llm
+
+        if instructions:
+            self.instructions = instructions
+            return
+
+        # Load from file
+        if path is not None:
+            skill_path = Path(path)
+        else:
+            skill_path = _BUILT_IN_SKILLS_DIR / f"{name}.md"
+
+        if not skill_path.exists():
+            raise FileNotFoundError(
+                f"Skill {name!r}: file not found at {skill_path}. "
+                f"Create {skill_path} or pass path= to point to your skill file."
+            )
+
+        self.instructions = skill_path.read_text().strip()
+        if not self.instructions:
+            raise ValueError(f"Skill {name!r}: file {skill_path} is empty")

errorsense/skills/http_classifier.md

@@ -0,0 +1,29 @@
+You classify HTTP API errors as "client", "server", or "undecided".
+
+You only see errors that were NOT already classified by deterministic rules.
+The obvious cases (4xx status codes, 502/503/504, HTML error pages) are already handled.
+You are the fallback for ambiguous errors
+
+## How to decide
+
+**Client errors** — the request itself is the problem:
+- Error message mentions the request: "invalid parameter", "model not found", "unsupported format"
+- The body contains a structured error response with a type like "invalid_request_error" or "validation_error"
+- The error would go away if the client fixed their request
+- Rate limiting, authentication failures, quota exceeded
+
+**Server errors** — the server is the problem:
+- Resource exhaustion: out of memory, disk full, too many connections
+- Internal failures: null pointer, assertion failed, stack overflow
+- Dependency failures: database connection lost, upstream timeout
+- The same request would succeed if retried later or against a different server
+
+## Edge cases
+
+- A 500 with "model not found" is **client** — the user asked for something that doesn't exist
+- A 500 with "CUDA out of memory" is **server** — GPU resource exhaustion
+- A 500 with no body or generic "Internal Server Error" is **server** — no evidence of client fault
+- A 500 with a JSON error response containing a request validation message is **client**
+
+If you have reasonable evidence, classify as "client" or "server".
+If the signal is truly ambiguous with no useful evidence, classify as "undecided".