typesafe-eval 1.0.0__py3-none-any.whl

typesafe_eval/__init__.py

@@ -0,0 +1,56 @@
+"""typesafe-eval: Fast, typed multi-dimensional document evaluation CLI using TypeSafe API (Jev)."""
+
+from typesafe_eval.api import evaluate, evaluate_document, evaluate_documents
+from typesafe_eval.cache import EvaluationCache
+from typesafe_eval.client import TypeSafeEvaluator
+from typesafe_eval.exceptions import (
+    AuthenticationError,
+    ConfigurationError,
+    ContentViolationError,
+    RuntimeEvalError,
+    TypeSafeEvalError,
+)
+from typesafe_eval.models import (
+    CANDIDATE_DECISION_THRESHOLD,
+    NEAR_THRESHOLD_MARGIN,
+    ChoiceResult,
+    DocumentEvalResult,
+    NoulResult,
+    PresetConfig,
+    QuestionConfig,
+    ScoreResult,
+    ViolationItem,
+)
+from typesafe_eval.presets import (
+    find_project_config,
+    load_preset,
+    load_project_config,
+)
+
+__version__ = "1.0.0"  # x-release-please-version
+
+__all__ = [
+    "__version__",
+    "evaluate",
+    "evaluate_document",
+    "evaluate_documents",
+    "TypeSafeEvaluator",
+    "EvaluationCache",
+    "DocumentEvalResult",
+    "ViolationItem",
+    "ScoreResult",
+    "NoulResult",
+    "ChoiceResult",
+    "PresetConfig",
+    "QuestionConfig",
+    "load_preset",
+    "load_project_config",
+    "find_project_config",
+    "TypeSafeEvalError",
+    "ConfigurationError",
+    "AuthenticationError",
+    "RuntimeEvalError",
+    "ContentViolationError",
+    "NEAR_THRESHOLD_MARGIN",
+    "CANDIDATE_DECISION_THRESHOLD",
+]

typesafe_eval/api.py

@@ -0,0 +1,247 @@
+"""High-level public Python API for typesafe-eval."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from typesafe_eval.client import TypeSafeEvaluator
+from typesafe_eval.exceptions import (
+    AuthenticationError,
+    ConfigurationError,
+    ContentViolationError,
+    RuntimeEvalError,
+    TypeSafeEvalError,
+)
+from typesafe_eval.models import DocumentEvalResult, PresetConfig
+from typesafe_eval.presets import is_path_excluded, load_preset, load_project_config
+
+
+def _resolve_preset(
+    preset: str | PresetConfig | None = "quality",
+    project_root: str | Path | None = None,
+) -> PresetConfig:
+    """Resolves a preset string, PresetConfig, or project configuration."""
+    if isinstance(preset, PresetConfig):
+        return preset
+
+    if preset is None or preset == "default":
+        start_dir = Path(project_root) if project_root else None
+        try:
+            discovered_cfg, _ = load_project_config(start_dir=start_dir)
+            if discovered_cfg is not None:
+                return discovered_cfg
+        except Exception as e:
+            raise ConfigurationError(f"Error loading discovered project configuration: {e}") from e
+        preset = "quality"
+
+    try:
+        return load_preset(preset)
+    except Exception as e:
+        raise ConfigurationError(f"Failed to load preset '{preset}': {e}") from e
+
+
+def evaluate(
+    content: str,
+    preset: str | PresetConfig = "quality",
+    *,
+    api_key: str | None = None,
+    dry_run: bool = False,
+    offline: bool = False,
+    cache: bool = True,
+    cache_dir: Path | str | None = None,
+    mask_secrets: bool = True,
+    max_chars: int = 120_000,
+    raise_on_violation: bool = False,
+    filename: str = "<memory>",
+    filepath: str | Path | None = None,
+    project_root: str | Path | None = None,
+    evaluator: TypeSafeEvaluator | None = None,
+) -> DocumentEvalResult:
+    """Evaluates raw in-memory content against a specified preset."""
+    if not isinstance(content, str):
+        raise ConfigurationError(f"Expected str content, got {type(content).__name__}")
+
+    preset_cfg = _resolve_preset(preset, project_root=project_root)
+
+    resolved_api_key = api_key or os.environ.get("TYPESAFE_API_KEY")
+    if not dry_run and not offline and not resolved_api_key:
+        raise AuthenticationError(
+            "No TypeSafe API key provided. Set the TYPESAFE_API_KEY environment variable "
+            "or pass api_key=..."
+        )
+
+    resolved_filepath = str(filepath) if filepath is not None else filename
+    active_evaluator = evaluator or TypeSafeEvaluator(
+        api_key=resolved_api_key, enable_cache=cache, cache_dir=cache_dir
+    )
+    try:
+        result = active_evaluator.evaluate_content(
+            content=content,
+            preset=preset_cfg,
+            filename=filename,
+            filepath=resolved_filepath,
+            mask_secrets=mask_secrets,
+            max_chars=max_chars,
+            dry_run=dry_run,
+            offline=offline,
+        )
+    except TypeSafeEvalError:
+        raise
+    except Exception as e:
+        raise RuntimeEvalError(f"Evaluation failed: {e}") from e
+
+    if raise_on_violation and not result.passed_thresholds:
+        raise ContentViolationError(
+            f"Content violation in '{filename}': {'; '.join(result.violations)}",
+            violations=result.violations,
+            result=result,
+            results=[result],
+        )
+
+    return result
+
+
+def evaluate_document(
+    path: str | Path,
+    preset: str | PresetConfig = "quality",
+    *,
+    api_key: str | None = None,
+    dry_run: bool = False,
+    offline: bool = False,
+    cache: bool = True,
+    cache_dir: Path | str | None = None,
+    mask_secrets: bool = True,
+    max_chars: int = 120_000,
+    raise_on_violation: bool = False,
+    project_root: str | Path | None = None,
+) -> DocumentEvalResult:
+    """Evaluates a single document file from disk against a specified preset."""
+    doc_path = Path(path)
+    if not doc_path.is_file():
+        raise ConfigurationError(f"Document file not found: {path}")
+
+    try:
+        content = doc_path.read_text(encoding="utf-8")
+    except Exception as e:
+        raise ConfigurationError(f"Failed to read file '{path}': {e}") from e
+
+    return evaluate(
+        content=content,
+        preset=preset,
+        api_key=api_key,
+        dry_run=dry_run,
+        offline=offline,
+        cache=cache,
+        cache_dir=cache_dir,
+        mask_secrets=mask_secrets,
+        max_chars=max_chars,
+        raise_on_violation=raise_on_violation,
+        filename=doc_path.name,
+        filepath=str(doc_path),
+        project_root=project_root or doc_path.parent,
+    )
+
+
+def evaluate_documents(
+    paths: Sequence[str | Path],
+    preset: str | PresetConfig = "quality",
+    *,
+    exclude: Sequence[str] | None = None,
+    concurrency: int = 4,
+    api_key: str | None = None,
+    dry_run: bool = False,
+    offline: bool = False,
+    cache: bool = True,
+    cache_dir: Path | str | None = None,
+    mask_secrets: bool = True,
+    max_chars: int = 120_000,
+    raise_on_violation: bool = False,
+    project_root: str | Path | None = None,
+) -> list[DocumentEvalResult]:
+    """Evaluates multiple document files concurrently preserving input order."""
+    if not paths:
+        return []
+
+    preset_cfg = _resolve_preset(preset, project_root=project_root)
+
+    resolved_api_key = api_key or os.environ.get("TYPESAFE_API_KEY")
+    if not dry_run and not offline and not resolved_api_key:
+        raise AuthenticationError(
+            "No TypeSafe API key provided. Set the TYPESAFE_API_KEY environment variable "
+            "or pass api_key=..."
+        )
+
+    # Combine exclusion patterns from argument and preset configuration
+    combined_excludes = list(exclude or []) + (preset_cfg.exclude or [])
+    root = Path(project_root) if project_root else Path.cwd()
+
+    # Validate all file paths upfront (skipping excluded)
+    resolved_paths: list[Path] = []
+    for p in paths:
+        doc_path = Path(p)
+        if combined_excludes and is_path_excluded(doc_path, combined_excludes, root=root):
+            continue
+        if not doc_path.is_file():
+            raise ConfigurationError(f"Document file not found: {p}")
+        resolved_paths.append(doc_path)
+
+    if not resolved_paths:
+        return []
+
+    # Shared evaluator for connection pooling across threads
+    shared_evaluator = TypeSafeEvaluator(
+        api_key=resolved_api_key, enable_cache=cache, cache_dir=cache_dir
+    )
+
+    def _eval_single(doc_p: Path) -> DocumentEvalResult:
+        try:
+            content = doc_p.read_text(encoding="utf-8")
+        except Exception as e:
+            raise ConfigurationError(f"Failed to read file '{doc_p}': {e}") from e
+
+        return evaluate(
+            content=content,
+            preset=preset_cfg,
+            api_key=resolved_api_key,
+            dry_run=dry_run,
+            offline=offline,
+            cache=cache,
+            cache_dir=cache_dir,
+            mask_secrets=mask_secrets,
+            max_chars=max_chars,
+            raise_on_violation=False,
+            filename=doc_p.name,
+            filepath=str(doc_p),
+            project_root=project_root or doc_p.parent,
+            evaluator=shared_evaluator,
+        )
+
+    concurrency = max(1, concurrency)
+    if concurrency == 1 or len(resolved_paths) == 1:
+        results = [_eval_single(p) for p in resolved_paths]
+    else:
+        with ThreadPoolExecutor(max_workers=min(concurrency, len(resolved_paths))) as executor:
+            results = list(executor.map(_eval_single, resolved_paths))
+
+    if raise_on_violation:
+        violated = [r for r in results if not r.passed_thresholds]
+        if violated:
+            all_violations = [v for r in violated for v in r.violations]
+            raise ContentViolationError(
+                f"{len(violated)} document(s) failed gates: {'; '.join(all_violations)}",
+                violations=all_violations,
+                result=violated[0] if len(violated) == 1 else None,
+                results=violated,
+            )
+
+    return results
+
+
+__all__ = [
+    "evaluate",
+    "evaluate_document",
+    "evaluate_documents",
+]

typesafe_eval/baseline.py

@@ -0,0 +1,218 @@
+"""Baseline diff mode: loads previous evaluation reports and detects regressions.
+
+Implements `--baseline` per Issue #39.
+"""
+
+import json
+from pathlib import Path
+
+from typesafe_eval.models import BaselineDiff, DocumentEvalResult, PresetConfig, QuestionDiff
+
+
+def _norm_preset(name: str | None) -> str:
+    return name.lower().replace("-", "_") if name else ""
+
+
+def _check_comparable(doc: DocumentEvalResult, preset_name: str | None) -> None:
+    """Raises ValueError if a baseline document cannot be compared with a run of preset_name.
+
+    preset_name=None skips the preset check (the dry-run check always applies).
+    """
+    if doc.mock or doc.model == "mock-jev":
+        raise ValueError(
+            f"Baseline contains dry-run document '{doc.filepath}' (mock=true or model 'mock-jev'); "
+            "cannot compare against dry-run baseline."
+        )
+    if preset_name is not None and _norm_preset(doc.preset_name) != _norm_preset(preset_name):
+        raise ValueError(
+            f"Baseline document '{doc.filepath}' has preset '{doc.preset_name}', "
+            f"differing from current run preset '{preset_name}'."
+        )
+
+
+def load_baseline(
+    baseline_path: str | Path,
+    expected_preset: str | None = None,
+) -> dict[str, DocumentEvalResult]:
+    """Loads a previous evaluation report JSON and returns an indexed lookup map."""
+    path = Path(baseline_path)
+    if not path.is_file():
+        raise FileNotFoundError(f"Baseline file not found: {path}")
+
+    raw_text = path.read_text(encoding="utf-8")
+    try:
+        data = json.loads(raw_text)
+    except Exception as e:
+        raise ValueError(f"Failed to parse baseline JSON {path}: {e}") from e
+
+    if isinstance(data, dict):
+        items = [data]
+    elif isinstance(data, list):
+        items = data
+    else:
+        raise ValueError(
+            f"Invalid baseline report structure in {path}: expected JSON list or object"
+        )
+
+    lookup: dict[str, DocumentEvalResult] = {}
+    for item in items:
+        try:
+            doc_res = DocumentEvalResult(**item)
+        except Exception as e:
+            raise ValueError(f"Invalid document result in baseline {path}: {e}") from e
+
+        _check_comparable(doc_res, expected_preset)
+
+        # Index by filepath and resolved path only (do not index by filename to prevent collision)
+        lookup[doc_res.filepath] = doc_res
+        try:
+            resolved_key = str(Path(doc_res.filepath).resolve())
+            lookup[resolved_key] = doc_res
+        except Exception:
+            pass
+
+    return lookup
+
+
+def compare_document_with_baseline(
+    result: DocumentEvalResult,
+    baseline_lookup: dict[str, DocumentEvalResult],
+    preset: PresetConfig,
+    default_max_drop: float = 0.10,
+) -> tuple[DocumentEvalResult, bool, str | None]:
+    """Compares a current document result with its baseline counterpart.
+
+    Returns (updated_result, has_regression, optional_truncation_warning).
+    """
+    # 1. Direct match by exact filepath or resolved absolute path
+    prev = None
+    for key in (result.filepath, str(Path(result.filepath).resolve())):
+        if key in baseline_lookup:
+            prev = baseline_lookup[key]
+            break
+
+    # 2. Portable cross-environment relative / suffix matching (e.g. CI runner absolute path vs local run)
+    if prev is None:
+        norm_target = Path(result.filepath).as_posix().lstrip("./")
+        seen_ids = set()
+        norm_docs = []
+        for doc in baseline_lookup.values():
+            if id(doc) not in seen_ids:
+                seen_ids.add(id(doc))
+                norm_docs.append((doc, Path(doc.filepath).as_posix().lstrip("./")))
+
+        # 2a. Prioritize exact normalized match
+        exact_matches = [doc for doc, doc_norm in norm_docs if doc_norm == norm_target]
+        if len(exact_matches) == 1:
+            prev = exact_matches[0]
+        elif not exact_matches:
+            # 2b. Fallback to suffix matching
+            suffix_matches = [
+                doc
+                for doc, doc_norm in norm_docs
+                if doc_norm.endswith("/" + norm_target) or norm_target.endswith("/" + doc_norm)
+            ]
+            if len(suffix_matches) == 1:
+                prev = suffix_matches[0]
+
+    if prev is None:
+        result.baseline_diff = BaselineDiff(status="new")
+        return result, False, None
+
+    _check_comparable(prev, preset.name)
+
+    # Check truncation mismatch
+    trunc_mismatch = prev.was_truncated != result.was_truncated
+    warning_msg = None
+    if trunc_mismatch:
+        warning_msg = (
+            f"Truncation status differs for {result.filename} "
+            f"(baseline was_truncated={prev.was_truncated}, current was_truncated={result.was_truncated}). "
+            f"Scores may be shifted by truncation."
+        )
+
+    has_regression = False
+    diff_questions: dict[str, QuestionDiff] = {}
+
+    for q_id, q_cfg in preset.questions.items():
+        threshold = q_cfg.max_drop if (q_cfg and q_cfg.max_drop is not None) else default_max_drop
+
+        curr_val: float | None = None
+        prev_val: float | None = None
+
+        if q_id in result.scores and q_id in prev.scores:
+            curr_val = result.scores[q_id].normalized_score
+            prev_val = prev.scores[q_id].normalized_score
+        elif q_id in result.nouls and q_id in prev.nouls:
+            if (
+                result.nouls[q_id].probability is not None
+                and prev.nouls[q_id].probability is not None
+            ):
+                curr_val = result.nouls[q_id].probability
+                prev_val = prev.nouls[q_id].probability
+
+        if curr_val is not None and prev_val is not None:
+            delta = curr_val - prev_val
+            is_risk_question = bool(q_cfg and q_cfg.max_threshold is not None)
+
+            if is_risk_question:
+                # For risk questions with max_threshold (e.g. has_secrets, has_pii, confidentiality_risk):
+                # A rise in probability/risk is a regression
+                rise = curr_val - prev_val
+                regressed = rise > threshold
+
+                diff_questions[q_id] = QuestionDiff(
+                    previous=prev_val,
+                    current=curr_val,
+                    delta=delta,
+                    max_drop=threshold,
+                    regressed=regressed,
+                )
+
+                is_advisory = bool(q_cfg and q_cfg.advisory)
+                if regressed:
+                    msg = (
+                        f"Baseline risk rise: '{q_id}' rose by {rise:.2f} "
+                        f"(prev: {prev_val:.2f}, curr: {curr_val:.2f}, max allowed rise: {threshold:.2f})"
+                    )
+                    if is_advisory:
+                        result.warnings.append(f"{msg} (warning)")
+                    else:
+                        has_regression = True
+                        result.passed_thresholds = False
+                        result.violations.append(msg)
+            else:
+                # For standard/min_threshold questions (e.g. clarity, completeness):
+                # A drop in score is a regression
+                drop = prev_val - curr_val
+                regressed = drop > threshold
+
+                diff_questions[q_id] = QuestionDiff(
+                    previous=prev_val,
+                    current=curr_val,
+                    delta=delta,
+                    max_drop=threshold,
+                    regressed=regressed,
+                )
+
+                is_advisory = bool(q_cfg and q_cfg.advisory)
+                if regressed:
+                    msg = (
+                        f"Baseline drop: '{q_id}' dropped by {drop:.2f} "
+                        f"(prev: {prev_val:.2f}, curr: {curr_val:.2f}, max allowed drop: {threshold:.2f})"
+                    )
+                    if is_advisory:
+                        result.warnings.append(f"{msg} (warning)")
+                    else:
+                        has_regression = True
+                        result.passed_thresholds = False
+                        result.violations.append(msg)
+
+    result.baseline_diff = BaselineDiff(
+        status="compared",
+        baseline_filepath=prev.filepath,
+        truncation_mismatch=trunc_mismatch,
+        questions=diff_questions,
+    )
+
+    return result, has_regression, warning_msg

typesafe_eval/cache.py

@@ -0,0 +1,156 @@
+"""Content-addressable evaluation result cache."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+
+from typesafe_eval.models import DocumentEvalResult, PresetConfig
+
+
+def _get_version() -> str:
+    try:
+        import typesafe_eval
+
+        return getattr(typesafe_eval, "__version__", "1.0.0")
+    except Exception:
+        return "1.0.0"
+
+
+def get_default_cache_dir() -> Path:
+    """Returns the default directory for caching evaluation results."""
+    env_dir = os.environ.get("TYPESAFE_CACHE_DIR")
+    if env_dir:
+        return Path(env_dir)
+
+    if os.environ.get("GITHUB_ACTIONS") == "true":
+        return Path(".typesafe-eval-cache")
+
+    # Standard XDG cache directory
+    xdg_cache = os.environ.get("XDG_CACHE_HOME")
+    if xdg_cache:
+        target = Path(xdg_cache) / "typesafe-eval"
+    else:
+        target = Path.home() / ".cache" / "typesafe-eval"
+
+    try:
+        target.mkdir(parents=True, exist_ok=True)
+        return target
+    except OSError:
+        # Fallback to local workspace cache directory if home/xdg cache is not writable
+        return Path(".typesafe-eval-cache")
+
+
+class EvaluationCache:
+    """Local file-based content-addressable cache for document evaluations."""
+
+    def __init__(
+        self,
+        cache_dir: Path | str | None = None,
+        enabled: bool = True,
+    ) -> None:
+        self.cache_dir = Path(cache_dir) if cache_dir else get_default_cache_dir()
+        self.enabled = enabled
+
+    def compute_key(
+        self,
+        content: str,
+        preset: PresetConfig,
+        mask_secrets: bool = True,
+        max_chars: int = 25000,
+    ) -> str:
+        """Computes a deterministic SHA-256 hash from document content, preset rules, and evaluation options."""
+        preset_repr = preset.model_dump_json()
+        hasher = hashlib.sha256()
+        hasher.update(content.encode())
+        hasher.update(b"::")
+        hasher.update(preset_repr.encode())
+        hasher.update(b"::")
+        hasher.update(f"mask={mask_secrets}:max={max_chars}".encode())
+        hasher.update(b"::")
+        hasher.update(_get_version().encode())
+        return hasher.hexdigest()
+
+    def get(
+        self,
+        content: str,
+        preset: PresetConfig,
+        mask_secrets: bool = True,
+        max_chars: int = 25000,
+    ) -> DocumentEvalResult | None:
+        """Retrieves a cached evaluation result if present and valid."""
+        if not self.enabled:
+            return None
+
+        key = self.compute_key(content, preset, mask_secrets=mask_secrets, max_chars=max_chars)
+        cache_file = self.cache_dir / f"{key}.json"
+
+        if not cache_file.is_file():
+            return None
+
+        try:
+            raw_text = cache_file.read_text(encoding="utf-8")
+            result = DocumentEvalResult.model_validate_json(raw_text)
+            result.cached = True
+            result.api_calls = 0
+            return result
+        except Exception:
+            # Corrupted cache file; safely remove and treat as miss
+            try:
+                cache_file.unlink(missing_ok=True)
+            except OSError:
+                pass
+            return None
+
+    def set(
+        self,
+        content: str,
+        preset: PresetConfig,
+        result: DocumentEvalResult,
+        mask_secrets: bool = True,
+        max_chars: int = 25000,
+    ) -> None:
+        """Stores an evaluation result in the cache atomically."""
+        if not self.enabled:
+            return
+
+        try:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            key = self.compute_key(content, preset, mask_secrets=mask_secrets, max_chars=max_chars)
+            target_path = self.cache_dir / f"{key}.json"
+            temp_path = self.cache_dir / f"{key}.tmp.{os.getpid()}"
+
+            # Clone result to avoid mutating caller object
+            res_dict = result.model_dump()
+            res_dict["cached"] = False  # Stored template is pristine
+            json_str = json.dumps(res_dict, ensure_ascii=False)
+
+            temp_path.write_text(json_str, encoding="utf-8")
+            temp_path.replace(target_path)
+        except Exception:
+            # Cache failure must never block application execution
+            pass
+
+    def clear(self) -> int:
+        """Clears all cached evaluations and returns the count of removed files."""
+        if not self.cache_dir.is_dir():
+            return 0
+
+        count = 0
+        try:
+            for item in self.cache_dir.glob("*.json"):
+                try:
+                    item.unlink()
+                    count += 1
+                except OSError:
+                    pass
+            for item in self.cache_dir.glob("*.tmp.*"):
+                try:
+                    item.unlink()
+                except OSError:
+                    pass
+        except OSError:
+            pass
+        return count