thinkless 0.2.0__py3-none-any.whl

thinkless/__init__.py

@@ -0,0 +1,63 @@
+"""ThinkLess: the decision plane for AI agents.
+
+Rules and small calibrated models make the routine decisions of an agent, a
+generative model is reserved for the steps that need thought, and every step
+is traced with its latency, confidence, tokens and cost.
+
+Quick start::
+
+    from thinkless import Choice, Engine
+    from thinkless.providers import GLiNER, LLMDecider, Rules
+    from thinkless.llm import TransformersLLM
+
+    llm = TransformersLLM("Qwen/Qwen3-1.7B")
+    engine = Engine([Rules(), GLiNER(), LLMDecider(llm)], llm=llm, threshold=0.8)
+
+    intent = engine.decide(
+        "I was charged twice for order #4471, please refund one of them.",
+        Choice("What does the customer want?", options={
+            "refund": "wants money back",
+            "order_status": "asks where an order is",
+            "other": "anything else",
+        }),
+    )
+    print(intent.value, intent.confidence, intent.provider)
+"""
+
+from ._version import __version__
+from .decision import Answer, Attempt, Decision, Plane, Status, Usage
+from .engine import Engine, Run
+from .errors import ConfigurationError, ThinkLessError
+from .logs import configure_logging
+from .questions import Choice, Extract, Kind, Question, Score, YesNo
+from .settings import load_env
+from .tracing import ConsoleSink, JSONLSink, MemorySink, Tracer, TraceSummary, summarize, tool
+
+__all__ = [
+    "Answer",
+    "Attempt",
+    "Choice",
+    "ConfigurationError",
+    "ConsoleSink",
+    "Decision",
+    "Engine",
+    "Extract",
+    "JSONLSink",
+    "Kind",
+    "MemorySink",
+    "Plane",
+    "Question",
+    "Run",
+    "Score",
+    "Status",
+    "ThinkLessError",
+    "TraceSummary",
+    "Tracer",
+    "Usage",
+    "YesNo",
+    "__version__",
+    "configure_logging",
+    "load_env",
+    "summarize",
+    "tool",
+]

thinkless/__main__.py

@@ -0,0 +1,3 @@
+from thinkless.cli.main import app
+
+app()

thinkless/_hub.py

@@ -0,0 +1,84 @@
+"""Model downloads from the Hugging Face Hub, made reliable on Windows."""
+
+from __future__ import annotations
+
+import os
+import platform
+from pathlib import Path
+
+from .logs import get_logger
+
+__all__ = ["DEFAULT_CHECKPOINTS", "cached_checkpoints", "ensure_downloaded"]
+
+logger = get_logger("hub")
+
+# What `thinkless demo` and the default providers download on first use.
+DEFAULT_CHECKPOINTS = {
+    "fastino/gliner2.5-base-v1": "GLiNER provider",
+    "convaiinnovations/laya": "Laya provider",
+    "Qwen/Qwen3-1.7B": "local LLM (--llm local)",
+}
+
+
+# Weight formats ThinkLess never loads. Skipping them keeps the serial download
+# as small as the one the model library would have made.
+IGNORED = [
+    "*.onnx",
+    "onnx/*",
+    "*.msgpack",
+    "*.h5",
+    "*.ot",
+    "*.tflite",
+    "*.gguf",
+    "flax_model*",
+    "tf_model*",
+    "rust_model*",
+    "coreml/*",
+    "openvino/*",
+]
+
+
+def _is_local_path(repo_id: str) -> bool:
+    return Path(repo_id).exists()
+
+
+def ensure_downloaded(repo_id: str) -> None:
+    """Download ``repo_id`` into the Hugging Face cache before a library loads it.
+
+    On Windows the Hub client downloads files in parallel and can hit a race
+    when it creates cache symlinks (``WinError 1314``). Downloading serially
+    first avoids it; the model library then finds everything in the cache.
+    Elsewhere this is a no-op and the library downloads as usual. Local paths
+    and offline mode (``HF_HUB_OFFLINE=1``) are left alone.
+    """
+    if platform.system() != "Windows" or _is_local_path(repo_id):
+        return
+    if os.environ.get("HF_HUB_OFFLINE", "").strip() in ("1", "true", "True"):
+        return
+    try:
+        from huggingface_hub import snapshot_download
+    except ImportError:  # pragma: no cover - installed with every local extra
+        return
+    try:
+        snapshot_download(repo_id, max_workers=1, ignore_patterns=IGNORED)
+    except Exception as exc:  # the library's own loader reports the real error
+        logger.warning(
+            "pre-download of %s failed (%s); falling back to the library loader", repo_id, exc
+        )
+
+
+def cached_checkpoints() -> dict[str, float | None]:
+    """Size in GB of each default checkpoint in the local cache, or ``None`` if absent."""
+    sizes: dict[str, float | None] = dict.fromkeys(DEFAULT_CHECKPOINTS)
+    try:
+        from huggingface_hub import scan_cache_dir
+    except ImportError:
+        return sizes
+    try:
+        info = scan_cache_dir()
+    except Exception:
+        return sizes
+    for repo in info.repos:
+        if repo.repo_id in sizes and repo.repo_type == "model":
+            sizes[repo.repo_id] = round(repo.size_on_disk / 1e9, 2)
+    return sizes

thinkless/_json.py

@@ -0,0 +1,33 @@
+"""JSON-safe conversion used by traces and reports."""
+
+from __future__ import annotations
+
+import dataclasses
+import enum
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+__all__ = ["jsonable"]
+
+
+def jsonable(value: Any, *, depth: int = 0) -> Any:
+    """Best-effort conversion of arbitrary values into JSON-safe data."""
+    if depth > 8:
+        return repr(value)
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return value
+    if isinstance(value, enum.Enum):
+        return value.value
+    if isinstance(value, BaseModel):
+        return jsonable(value.model_dump(mode="json"), depth=depth + 1)
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return jsonable(dataclasses.asdict(value), depth=depth + 1)
+    if isinstance(value, dict):
+        return {str(k): jsonable(v, depth=depth + 1) for k, v in value.items()}
+    if isinstance(value, (list, tuple, set, frozenset)):
+        return [jsonable(v, depth=depth + 1) for v in value]
+    if isinstance(value, Path):
+        return str(value)
+    return repr(value)

thinkless/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

thinkless/bench/__init__.py

@@ -0,0 +1,22 @@
+"""Benchmarks and calibration."""
+
+from .metrics import (
+    ThresholdPoint,
+    expected_calibration_error,
+    percentile,
+    recommend_threshold,
+    threshold_sweep,
+)
+from .support import ModeReport, SupportBenchmark, TicketResult, run_support_benchmark
+
+__all__ = [
+    "ModeReport",
+    "SupportBenchmark",
+    "ThresholdPoint",
+    "TicketResult",
+    "expected_calibration_error",
+    "percentile",
+    "recommend_threshold",
+    "run_support_benchmark",
+    "threshold_sweep",
+]

thinkless/bench/intents.py

@@ -0,0 +1,406 @@
+"""Intent classification benchmark on public datasets.
+
+Measures each provider's accuracy, calibration and latency on one ``Choice``
+question, then simulates the cascade: at every threshold, how often the small
+models answer on their own, how accurate the combined system is, and how many
+calls reach the LLM.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import random
+import time
+from collections.abc import Callable, Sequence
+from pathlib import Path
+from typing import Any
+
+import httpx
+from pydantic import BaseModel, Field
+
+from .._version import __version__
+from ..engine import Engine
+from ..providers.base import DecisionProvider
+from ..questions import Choice, YesNo
+from ..tracing import Tracer
+from .metrics import (
+    ThresholdPoint,
+    expected_calibration_error,
+    percentile,
+    recommend_threshold,
+    threshold_sweep,
+)
+from .support import environment_info
+
+__all__ = [
+    "DATASETS",
+    "CascadePoint",
+    "ChoiceEvaluation",
+    "IntentsBenchmark",
+    "ProviderEval",
+    "build_provider",
+    "evaluate_choice",
+    "evaluate_question",
+    "load_dataset_rows",
+    "run_intents_benchmark",
+]
+
+BANKING77_TEST = "https://raw.githubusercontent.com/PolyAI-LDN/task-specific-datasets/master/banking_data/test.csv"
+
+DATASETS: dict[str, dict[str, Any]] = {
+    "banking77": {
+        "title": "Banking77 (PolyAI), 77 banking intents",
+        "instructions": "What is the customer's banking request about?",
+        "source": "https://github.com/PolyAI-LDN/task-specific-datasets",
+    },
+    "clinc150": {
+        "title": "CLINC150 plus (clinc/clinc_oos), 150 intents and out of scope",
+        "instructions": "What does the user want?",
+        "source": "https://huggingface.co/datasets/clinc/clinc_oos",
+        "repo": "clinc/clinc_oos",
+        "config": "plus",
+        "label": "intent",
+    },
+    "emotion": {
+        "title": "Emotion (dair-ai/emotion), 6 emotions",
+        "instructions": "Which emotion does the text express?",
+        "source": "https://huggingface.co/datasets/dair-ai/emotion",
+        "repo": "dair-ai/emotion",
+        "config": "split",
+        "label": "label",
+    },
+}
+
+
+def _humanize(label: str) -> str:
+    return "out of scope" if label == "oos" else label.replace("_", " ")
+
+
+def _cache_dir() -> Path:
+    path = Path.home() / ".cache" / "thinkless" / "datasets"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def load_dataset_rows(name: str) -> list[dict[str, str]]:
+    """Test split of a supported dataset as ``{"text", "label"}`` rows with readable labels."""
+    if name not in DATASETS:
+        raise ValueError(f"unknown dataset {name!r}; choose one of {', '.join(DATASETS)}")
+    if name == "banking77":
+        cached = _cache_dir() / "banking77-test.csv"
+        if not cached.exists():
+            response = httpx.get(BANKING77_TEST, timeout=60.0, follow_redirects=True)
+            response.raise_for_status()
+            cached.write_text(response.text, encoding="utf-8")
+        reader = csv.DictReader(io.StringIO(cached.read_text(encoding="utf-8")))
+        return [{"text": row["text"], "label": _humanize(row["category"])} for row in reader]
+    try:
+        from datasets import load_dataset
+    except ImportError as exc:  # pragma: no cover - depends on the extra
+        raise ImportError(
+            f'The {name} dataset needs the bench extra: pip install "thinkless[bench]"'
+        ) from exc
+    spec = DATASETS[name]
+    dataset = load_dataset(spec["repo"], spec["config"], split="test")
+    names = dataset.features[spec["label"]].names
+    return [{"text": row["text"], "label": _humanize(names[row[spec["label"]]])} for row in dataset]
+
+
+def build_provider(
+    name: str, *, llm_spec: str = "local", device: str = "auto", reasoning: str | None = None
+) -> DecisionProvider:
+    """A provider by short name: ``gliner``, ``laya``, ``llm`` or ``jev``."""
+    if name == "gliner":
+        from ..providers.gliner import GLiNER
+
+        return GLiNER(device=device)
+    if name == "laya":
+        from ..providers.laya import Laya
+
+        return Laya(device=device)
+    if name == "llm":
+        from ..llm import from_spec
+        from ..providers import LLMDecider
+
+        return LLMDecider(from_spec(llm_spec, device=device, reasoning=reasoning))
+    if name == "jev":
+        from ..providers import SystemOne
+
+        return SystemOne.jev()
+    raise ValueError(f"unknown provider {name!r}; choose gliner, laya, llm or jev")
+
+
+class ChoiceEvaluation(BaseModel):
+    """One provider's answers to one question over a labeled set."""
+
+    provider: str
+    predictions: list[Any]
+    confidences: list[float | None]
+    correct: list[bool]
+    latencies_ms: list[float]
+    input_tokens: list[int]
+    costs_usd: list[float] = Field(default_factory=list)
+
+    @property
+    def accuracy(self) -> float:
+        return sum(self.correct) / len(self.correct) if self.correct else 0.0
+
+    @property
+    def latency_p50_ms(self) -> float:
+        return percentile(self.latencies_ms, 50)
+
+
+def evaluate_question(
+    provider: DecisionProvider,
+    question: Choice | YesNo,
+    rows: Sequence[dict[str, Any]],
+    *,
+    text_field: str = "text",
+    label_field: str = "label",
+    progress: Callable[[int, int], None] | None = None,
+) -> ChoiceEvaluation:
+    """Ask ``question`` about every row and score the answer against the row's label.
+
+    For a ``Choice`` the label is an option; for a ``YesNo`` it is a boolean.
+    """
+    provider.warmup()
+    engine = Engine([provider], threshold=0.0, tracer=Tracer())
+    predictions: list[Any] = []
+    confidences: list[float | None] = []
+    correct: list[bool] = []
+    latencies: list[float] = []
+    tokens: list[int] = []
+    costs: list[float] = []
+    for index, row in enumerate(rows):
+        truth = row[label_field]
+        if isinstance(question, YesNo):
+            truth = (
+                truth
+                if isinstance(truth, bool)
+                else str(truth).strip().lower() in ("true", "yes", "1")
+            )
+        started = time.perf_counter()
+        decision = engine.decide(row[text_field], question)
+        latencies.append((time.perf_counter() - started) * 1000.0)
+        predictions.append(decision.value)
+        confidences.append(decision.confidence)
+        correct.append(decision.value == truth)
+        tokens.append(decision.usage.input_tokens)
+        costs.append(decision.cost_usd)
+        if progress is not None:
+            progress(index + 1, len(rows))
+    return ChoiceEvaluation(
+        provider=provider.name,
+        predictions=predictions,
+        confidences=confidences,
+        correct=correct,
+        latencies_ms=latencies,
+        input_tokens=tokens,
+        costs_usd=costs,
+    )
+
+
+def evaluate_choice(
+    provider: DecisionProvider,
+    question: Choice,
+    rows: Sequence[dict[str, Any]],
+    *,
+    progress: Callable[[int, int], None] | None = None,
+) -> ChoiceEvaluation:
+    """Shorthand for :func:`evaluate_question` on ``{"text", "label"}`` rows."""
+    return evaluate_question(provider, question, rows, progress=progress)
+
+
+class ProviderEval(BaseModel):
+    provider: str
+    accuracy: float
+    ece: float | None
+    latency_p50_ms: float
+    latency_p95_ms: float
+    mean_input_tokens: float
+    cost_per_1k: float = 0.0
+    abstained: int
+    sweep: list[ThresholdPoint] = Field(default_factory=list)
+    recommended: ThresholdPoint | None = None
+
+
+class CascadePoint(BaseModel):
+    """The simulated cascade (small models in order, then the LLM) at one threshold."""
+
+    threshold: float
+    accuracy: float
+    llm_share: float
+    mean_latency_ms: float
+    cost_per_1k: float = 0.0
+    answered_by: dict[str, float]
+
+
+class IntentsBenchmark(BaseModel):
+    thinkless_version: str = __version__
+    created_at: str
+    dataset: str
+    dataset_title: str
+    source: str
+    examples: int
+    labels: int
+    seed: int
+    llm: str
+    target_accuracy: float
+    environment: dict[str, Any]
+    providers: dict[str, ProviderEval]
+    cascade: list[CascadePoint]
+
+
+def _cost_at(evaluation: ChoiceEvaluation, index: int) -> float:
+    return evaluation.costs_usd[index] if index < len(evaluation.costs_usd) else 0.0
+
+
+def _cascade(
+    evaluations: dict[str, ChoiceEvaluation], order: Sequence[str], fallback: str | None
+) -> list[CascadePoint]:
+    small = [p for p in order if p != fallback]
+    n = len(next(iter(evaluations.values())).correct)
+    points = []
+    for threshold in [round(x * 0.05, 2) for x in range(0, 20)] + [0.97, 0.99, 1.01]:
+        hits = 0
+        llm_calls = 0
+        latency = 0.0
+        cost = 0.0
+        answered: dict[str, int] = dict.fromkeys([*small, fallback or "unresolved"], 0)
+        for i in range(n):
+            chosen = None
+            for name in small:
+                latency += evaluations[name].latencies_ms[i]
+                cost += _cost_at(evaluations[name], i)
+                confidence = evaluations[name].confidences[i]
+                if confidence is not None and confidence >= threshold:
+                    chosen = name
+                    break
+            if chosen is None and fallback is not None:
+                chosen = fallback
+                llm_calls += 1
+                latency += evaluations[fallback].latencies_ms[i]
+                cost += _cost_at(evaluations[fallback], i)
+            if chosen is None:
+                answered["unresolved"] += 1
+                continue
+            answered[chosen] += 1
+            hits += int(evaluations[chosen].correct[i])
+        points.append(
+            CascadePoint(
+                threshold=threshold,
+                accuracy=round(hits / n, 4),
+                llm_share=round(llm_calls / n, 4),
+                mean_latency_ms=round(latency / n, 2),
+                cost_per_1k=round(cost / n * 1000, 5),
+                answered_by={k: round(v / n, 4) for k, v in answered.items()},
+            )
+        )
+    return points
+
+
+def _resolved_llm(spec: str) -> str:
+    from ..llm.factory import DEFAULT_LOCAL_MODEL
+
+    backend, _, model = spec.partition(":")
+    if backend == "local" and not model:
+        return f"local:{DEFAULT_LOCAL_MODEL}"
+    return spec
+
+
+def run_intents_benchmark(
+    dataset: str,
+    *,
+    providers: Sequence[str] = ("gliner", "laya", "llm"),
+    llm_spec: str = "local",
+    device: str = "auto",
+    reasoning: str | None = None,
+    limit: int = 500,
+    seed: int = 13,
+    target_accuracy: float = 0.95,
+    output_dir: str | Path | None = None,
+    progress: Callable[[str], None] | None = None,
+) -> IntentsBenchmark:
+    """Evaluate providers on a dataset sample and simulate the cascade.
+
+    The sample is drawn with a fixed seed so runs are comparable. When ``llm``
+    is among the providers it acts as the cascade's fallback.
+    """
+    say = progress or (lambda message: None)
+    rows = load_dataset_rows(dataset)
+    rng = random.Random(seed)
+    sample = rng.sample(rows, min(limit, len(rows)))
+    labels = sorted({row["label"] for row in rows})
+    question = Choice(DATASETS[dataset]["instructions"], options=labels, name="label")
+    say(f"{dataset}: {len(sample)} of {len(rows)} test examples, {len(labels)} labels")
+
+    evaluations: dict[str, ChoiceEvaluation] = {}
+    reports: dict[str, ProviderEval] = {}
+    for name in providers:
+        provider = build_provider(name, llm_spec=llm_spec, device=device, reasoning=reasoning)
+        started = time.perf_counter()
+
+        def tick(done: int, total: int, name: str = name, started: float = started) -> None:
+            if done % 50 == 0 or done == total:
+                say(f"{name}: {done}/{total}  {time.perf_counter() - started:.0f}s")
+
+        evaluation = evaluate_choice(provider, question, sample, progress=tick)
+        evaluations[name] = evaluation
+        calibrated = all(c is not None for c in evaluation.confidences)
+        confidences = [c if c is not None else 1.0 for c in evaluation.confidences]
+        sweep = threshold_sweep(confidences, evaluation.correct) if calibrated else []
+        reports[name] = ProviderEval(
+            provider=name,
+            accuracy=round(evaluation.accuracy, 4),
+            ece=round(expected_calibration_error(confidences, evaluation.correct), 4)
+            if calibrated
+            else None,
+            latency_p50_ms=round(evaluation.latency_p50_ms, 2),
+            latency_p95_ms=round(percentile(evaluation.latencies_ms, 95), 2),
+            mean_input_tokens=round(sum(evaluation.input_tokens) / len(sample), 1),
+            cost_per_1k=round(sum(evaluation.costs_usd) / len(sample) * 1000, 5),
+            abstained=sum(p is None for p in evaluation.predictions),
+            sweep=sweep,
+            recommended=recommend_threshold(sweep, target_accuracy) if sweep else None,
+        )
+        provider.close()
+
+    fallback = "llm" if "llm" in providers else None
+    result = IntentsBenchmark(
+        created_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        dataset=dataset,
+        dataset_title=DATASETS[dataset]["title"],
+        source=DATASETS[dataset]["source"],
+        examples=len(sample),
+        labels=len(labels),
+        seed=seed,
+        llm=(
+            _resolved_llm(llm_spec)
+            + ("" if reasoning in (None, "default") else f" (reasoning {reasoning})")
+        )
+        if "llm" in providers
+        else "not used",
+        target_accuracy=target_accuracy,
+        environment=environment_info(),
+        providers=reports,
+        cascade=_cascade(evaluations, list(providers), fallback),
+    )
+    if output_dir is not None:
+        out = Path(output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "results.json").write_text(
+            json.dumps(result.model_dump(mode="json"), indent=2), encoding="utf-8"
+        )
+        with (out / "predictions.jsonl").open("w", encoding="utf-8") as handle:
+            for i, row in enumerate(sample):
+                record: dict[str, Any] = {"text": row["text"], "label": row["label"]}
+                for name, evaluation in evaluations.items():
+                    record[name] = {
+                        "prediction": evaluation.predictions[i],
+                        "confidence": evaluation.confidences[i],
+                        "latency_ms": round(evaluation.latencies_ms[i], 2),
+                    }
+                handle.write(json.dumps(record, ensure_ascii=False) + "\n")
+    return result