ifixai 3.0.2__py3-none-any.whl

ifixai/__init__.py

@@ -0,0 +1,3 @@
+from ifixai._version import VERSION as __version__
+
+__all__ = ["__version__"]

ifixai/_version.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import re
+from importlib import metadata
+from pathlib import Path
+from typing import Final
+
+
+def _read_pyproject_version_fallback() -> str | None:
+    candidate = Path(__file__).resolve().parent.parent / "pyproject.toml"
+    if not candidate.exists():
+        return None
+    try:
+        text = candidate.read_text(encoding="utf-8")
+    except OSError:
+        return None
+    match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE)
+    return match.group(1) if match else None
+
+
+def _resolve_version() -> str:
+    try:
+        return metadata.version("ifixai")
+    except metadata.PackageNotFoundError:
+        fallback = _read_pyproject_version_fallback()
+        return fallback or "unknown"
+
+
+VERSION: Final[str] = _resolve_version()

ifixai/api.py

@@ -0,0 +1,310 @@
+import logging
+
+from ifixai._version import VERSION as __version__  # noqa: F401
+from ifixai.harness.registry import ALL_SPECS
+from ifixai.core.concurrency import ConcurrencyGovernor
+from ifixai.core.fixture_loader import list_fixture_names, load_fixture
+from ifixai.inspections.holdout_ids import generate_holdout_ids
+from ifixai.judge.config import JudgeConfig
+from ifixai.providers.base import ChatProvider
+from ifixai.providers.resolver import resolve_provider, wrap_with_governance
+from ifixai.reporting.comparison import compare_scorecards as _compare_scorecards
+from ifixai.core.runner import (
+    run_all,
+    run_selected as _run_selected,
+    run_single as _run_single,
+    run_strategic as _run_strategic,
+)
+from ifixai.core.types import (
+    TestResult,
+    InspectionSpec,
+    ComparisonReport,
+    EvaluationPipelineConfig,
+    Fixture,
+    ProviderConfig,
+    TestRunResult,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+def _resolve_fixture(fixture: str | Fixture) -> Fixture:
+    if isinstance(fixture, Fixture):
+        return fixture
+    return load_fixture(fixture)
+
+
+def _resolve_provider_with_governance(
+    provider: str | ChatProvider,
+    fixture_obj: Fixture,
+) -> ChatProvider:
+    """Resolve a provider and compose the fixture's governance bundle on it.
+
+    Without this composition, structural inspections (B02, B04, B11, B23,
+    B26, B27, B28) hit the base ChatProvider methods that return None and
+    every governance test is reported as INCONCLUSIVE. wrap_with_governance
+    is idempotent: calling it on an already-wrapped instance just refreshes
+    the bound governance fixture.
+    """
+    provider_obj = resolve_provider(provider)
+    if fixture_obj.governance is not None:
+        provider_obj = wrap_with_governance(provider_obj, fixture_obj.governance)
+    return provider_obj
+
+
+def _build_config(
+    provider: str | ChatProvider,
+    api_key: str,
+    endpoint: str | None,
+    model: str | None,
+    system_prompt: str | None,
+    timeout: int,
+    max_retries: int,
+    temperature: float = 0.0,
+    seed: int | None = None,
+    run_nonce: str | None = None,
+    holdout_ids: dict[str, str] | None = None,
+) -> ProviderConfig:
+    # holdout_ids must be populated for B01/B04/B16 to run; callers (CLI) can
+    # supply their own (e.g. for manifest-replay), otherwise auto-generate per
+    # run so direct API use does not fail with ConfigError.
+    resolved_holdout = (
+        holdout_ids if holdout_ids is not None else generate_holdout_ids().to_dict()
+    )
+    return ProviderConfig(
+        provider=provider if isinstance(provider, str) else "custom",
+        endpoint=endpoint,
+        api_key=api_key,
+        model=model,
+        system_prompt=system_prompt,
+        timeout=timeout,
+        max_retries=max_retries,
+        temperature=temperature,
+        seed=seed,
+        run_nonce=run_nonce,
+        holdout_ids=resolved_holdout,
+    )
+
+
+async def _aclose_provider(provider_obj: ChatProvider) -> None:
+    """Best-effort teardown for the SUT provider's shared HTTP/SDK pool.
+
+    Called from the api-level try/finally so the provider's connection
+    pool is closed even when an inspection raises mid-run. We log and
+    swallow because teardown failures must not mask the original error
+    that the caller is propagating.
+    """
+    try:
+        await provider_obj.aclose()
+    except Exception:
+        _logger.exception("Provider teardown failed during aclose")
+
+
+async def run_inspections(
+    provider: str | ChatProvider,
+    fixture: str | Fixture = "default",
+    api_key: str = "",
+    system_name: str = "",
+    system_version: str = "1.0",
+    endpoint: str | None = None,
+    model: str | None = None,
+    system_prompt: str | None = None,
+    timeout: int = 30,
+    max_retries: int = 3,
+    progress_callback: object = None,
+    pipeline_config: EvaluationPipelineConfig | None = None,
+    judge_config: JudgeConfig | None = None,
+    governor: "ConcurrencyGovernor | None" = None,
+    sut_temperature: float = 0.0,
+    sut_seed: int | None = None,
+    run_nonce: str | None = None,
+    holdout_ids: dict[str, str] | None = None,
+) -> TestRunResult:
+    fixture_obj = _resolve_fixture(fixture)
+    provider_obj = _resolve_provider_with_governance(provider, fixture_obj)
+    try:
+        return await run_all(
+            provider=provider_obj,
+            config=_build_config(
+                provider,
+                api_key,
+                endpoint,
+                model,
+                system_prompt,
+                timeout,
+                max_retries,
+                sut_temperature,
+                sut_seed,
+                run_nonce,
+                holdout_ids,
+            ),
+            fixture=fixture_obj,
+            system_name=system_name,
+            system_version=system_version,
+            progress_callback=progress_callback,
+            judge_config=judge_config,
+            pipeline_config=pipeline_config,
+            governor=governor,
+        )
+    finally:
+        await _aclose_provider(provider_obj)
+
+
+async def run_strategic(
+    provider: str | ChatProvider,
+    fixture: str | Fixture = "default",
+    api_key: str = "",
+    system_name: str = "",
+    system_version: str = "1.0",
+    endpoint: str | None = None,
+    model: str | None = None,
+    system_prompt: str | None = None,
+    timeout: int = 30,
+    max_retries: int = 3,
+    progress_callback: object = None,
+    pipeline_config: EvaluationPipelineConfig | None = None,
+    judge_config: JudgeConfig | None = None,
+    governor: "ConcurrencyGovernor | None" = None,
+    sut_temperature: float = 0.0,
+    sut_seed: int | None = None,
+    run_nonce: str | None = None,
+    holdout_ids: dict[str, str] | None = None,
+) -> TestRunResult:
+    fixture_obj = _resolve_fixture(fixture)
+    provider_obj = _resolve_provider_with_governance(provider, fixture_obj)
+    try:
+        return await _run_strategic(
+            provider=provider_obj,
+            config=_build_config(
+                provider,
+                api_key,
+                endpoint,
+                model,
+                system_prompt,
+                timeout,
+                max_retries,
+                sut_temperature,
+                sut_seed,
+                run_nonce,
+                holdout_ids,
+            ),
+            fixture=fixture_obj,
+            system_name=system_name,
+            system_version=system_version,
+            progress_callback=progress_callback,
+            judge_config=judge_config,
+            pipeline_config=pipeline_config,
+            governor=governor,
+        )
+    finally:
+        await _aclose_provider(provider_obj)
+
+
+async def run_selected(
+    test_ids: set[str],
+    provider: str | ChatProvider,
+    fixture: str | Fixture = "default",
+    api_key: str = "",
+    system_name: str = "",
+    system_version: str = "1.0",
+    endpoint: str | None = None,
+    model: str | None = None,
+    system_prompt: str | None = None,
+    timeout: int = 30,
+    max_retries: int = 3,
+    progress_callback: object = None,
+    pipeline_config: EvaluationPipelineConfig | None = None,
+    judge_config: JudgeConfig | None = None,
+    governor: "ConcurrencyGovernor | None" = None,
+    sut_temperature: float = 0.0,
+    sut_seed: int | None = None,
+    run_nonce: str | None = None,
+    holdout_ids: dict[str, str] | None = None,
+) -> TestRunResult:
+    fixture_obj = _resolve_fixture(fixture)
+    provider_obj = _resolve_provider_with_governance(provider, fixture_obj)
+    try:
+        return await _run_selected(
+            test_ids=test_ids,
+            provider=provider_obj,
+            config=_build_config(
+                provider,
+                api_key,
+                endpoint,
+                model,
+                system_prompt,
+                timeout,
+                max_retries,
+                sut_temperature,
+                sut_seed,
+                run_nonce,
+                holdout_ids,
+            ),
+            fixture=fixture_obj,
+            system_name=system_name,
+            system_version=system_version,
+            progress_callback=progress_callback,
+            judge_config=judge_config,
+            pipeline_config=pipeline_config,
+            governor=governor,
+        )
+    finally:
+        await _aclose_provider(provider_obj)
+
+
+async def run_single(
+    test_id: str,
+    provider: str | ChatProvider,
+    fixture: str | Fixture = "default",
+    api_key: str = "",
+    endpoint: str | None = None,
+    model: str | None = None,
+    system_prompt: str | None = None,
+    timeout: int = 30,
+    max_retries: int = 3,
+    pipeline_config: EvaluationPipelineConfig | None = None,
+    judge_config: JudgeConfig | None = None,
+    sut_temperature: float = 0.0,
+    sut_seed: int | None = None,
+    run_nonce: str | None = None,
+    holdout_ids: dict[str, str] | None = None,
+) -> TestResult:
+    fixture_obj = _resolve_fixture(fixture)
+    provider_obj = _resolve_provider_with_governance(provider, fixture_obj)
+    try:
+        return await _run_single(
+            test_id=test_id,
+            provider=provider_obj,
+            config=_build_config(
+                provider,
+                api_key,
+                endpoint,
+                model,
+                system_prompt,
+                timeout,
+                max_retries,
+                sut_temperature,
+                sut_seed,
+                run_nonce,
+                holdout_ids,
+            ),
+            fixture=fixture_obj,
+            judge_config=judge_config,
+            pipeline_config=pipeline_config,
+        )
+    finally:
+        await _aclose_provider(provider_obj)
+
+
+def compare_scorecards(
+    baseline: TestRunResult, enhanced: TestRunResult
+) -> ComparisonReport:
+    return _compare_scorecards(baseline, enhanced)
+
+
+def list_tests() -> list[InspectionSpec]:
+    return list(ALL_SPECS)
+
+
+def list_fixtures() -> list[str]:
+    return list_fixture_names()

ifixai/cli/_branding.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Iterable
+
+import click
+
+from ifixai.core.types import InspectionCategory, InspectionSpec
+
+_SPINNER_FRAMES = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
+
+
+class _Spinner:
+    def __init__(self, message: str) -> None:
+        self._message = message
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+        self._frame = 0
+        self._start_time = 0.0
+
+    def start(self) -> None:
+        if self._thread is not None:
+            return
+        self._start_time = time.monotonic()
+        sys.stdout.write(self._render() + "\n")
+        sys.stdout.flush()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        if self._thread is None:
+            return
+        self._stop_event.set()
+        self._thread.join(timeout=0.5)
+        self._thread = None
+
+    def _render(self) -> str:
+        glyph = _SPINNER_FRAMES[self._frame % len(_SPINNER_FRAMES)]
+        elapsed = int(time.monotonic() - self._start_time) if self._start_time else 0
+        suffix = f"  ({elapsed}s)" if elapsed >= 3 else ""
+        return _truecolor(f"  {glyph} {self._message}{suffix}", _DIM_RGB)
+
+    def _run(self) -> None:
+        while not self._stop_event.wait(0.12):
+            self._frame += 1
+            sys.stdout.write("\033[1F\033[2K" + self._render() + "\n")
+            sys.stdout.flush()
+
+
+_LOGO_LINES: tuple[str, ...] = (
+    "██  ███████ ██ ██   ██   █████  ██",
+    "██  ██      ██  ██ ██   ██   ██ ██",
+    "██  █████   ██   ███    ███████ ██",
+    "██  ██      ██  ██ ██   ██   ██ ██",
+    "██  ██      ██ ██   ██  ██   ██ ██",
+)
+
+_ACCENT_RGB = (232, 99, 42)
+_DIM_RGB = (110, 110, 117)
+
+_CATEGORY_COLORS: dict[InspectionCategory, tuple[int, int, int]] = {
+    InspectionCategory.FABRICATION: (255, 139, 92),
+    InspectionCategory.MANIPULATION: (255, 99, 99),
+    InspectionCategory.DECEPTION: (167, 139, 250),
+    InspectionCategory.UNPREDICTABILITY: (251, 191, 36),
+    InspectionCategory.OPACITY: (96, 165, 250),
+}
+
+_CATEGORY_ORDER: tuple[InspectionCategory, ...] = (
+    InspectionCategory.FABRICATION,
+    InspectionCategory.MANIPULATION,
+    InspectionCategory.DECEPTION,
+    InspectionCategory.UNPREDICTABILITY,
+    InspectionCategory.OPACITY,
+)
+
+_BAR_WIDTH = 26
+_BLOCK_FULL = "█"
+_BLOCK_EMPTY = "·"
+
+
+def supports_color(stream=None) -> bool:
+    s = stream or sys.stdout
+    if os.environ.get("NO_COLOR"):
+        return False
+    if not hasattr(s, "isatty"):
+        return False
+    return bool(s.isatty())
+
+
+def _truecolor(text: str, rgb: tuple[int, int, int], bold: bool = False) -> str:
+    if not supports_color():
+        return text
+    r, g, b = rgb
+    prefix = f"\033[38;2;{r};{g};{b}m"
+    if bold:
+        prefix = "\033[1m" + prefix
+    return f"{prefix}{text}\033[0m"
+
+
+def print_startup_banner(version: str, *, quiet: bool = False) -> None:
+    if quiet or not supports_color():
+        return
+    click.echo()
+    for line in _LOGO_LINES:
+        click.echo("  " + _truecolor(line, _ACCENT_RGB, bold=True))
+    click.echo()
+    click.echo(_truecolor(f"  ™  ·  v{version}  ·  powered by iMe", _DIM_RGB))
+    click.echo()
+
+
+@dataclass
+class _CategoryRow:
+    category: InspectionCategory
+    total: int = 0
+    done: int = 0
+    failed: int = 0
+
+
+@dataclass
+class CategoryProgress:
+    rows: dict[InspectionCategory, _CategoryRow] = field(default_factory=dict)
+    _printed_lines: int = 0
+    _started: bool = False
+    _interactive: bool = False
+    _spinner: _Spinner | None = None
+
+    @classmethod
+    def from_totals(cls, totals: dict[InspectionCategory, int]) -> "CategoryProgress":
+        rows = {
+            cat: _CategoryRow(category=cat, total=totals.get(cat, 0))
+            for cat in _CATEGORY_ORDER
+        }
+        return cls(rows=rows)
+
+    def start(self) -> None:
+        if self._started:
+            return
+        self._started = True
+        self._interactive = supports_color()
+        if not self._interactive:
+            return
+        total = sum(r.total for r in self.rows.values())
+        if total == 0:
+            return
+        self._spinner = _Spinner(f"Running {total} tests")
+        self._spinner.start()
+        self._printed_lines = 1
+
+    def record(self, category: InspectionCategory, passing: bool) -> None:
+        row = self.rows.get(category)
+        if row is None:
+            return
+        if row.total == 0:
+            row.total = max(1, row.total)
+        row.done += 1
+        if not passing:
+            row.failed += 1
+        self._redraw()
+
+    def finalize(self) -> None:
+        if self._spinner is not None:
+            self._spinner.stop()
+            self._spinner = None
+        self._redraw(final=True)
+        if self._interactive and self._printed_lines:
+            click.echo()
+
+    def _redraw(self, *, final: bool = False) -> None:
+        if not self._started:
+            return
+        rendered = self._render()
+        if not rendered:
+            return
+        if self._spinner is not None:
+            self._spinner.stop()
+            self._spinner = None
+        if self._interactive and self._printed_lines:
+            sys.stdout.write(f"\033[{self._printed_lines}F")
+            for line in rendered:
+                sys.stdout.write("\033[2K")
+                sys.stdout.write(line + "\n")
+            sys.stdout.flush()
+            self._printed_lines = len(rendered)
+        elif self._interactive and not self._printed_lines:
+            for line in rendered:
+                sys.stdout.write(line + "\n")
+            sys.stdout.flush()
+            self._printed_lines = len(rendered)
+        elif not self._interactive and final:
+            for line in rendered:
+                click.echo(line)
+
+    def _render(self) -> list[str]:
+        out: list[str] = []
+        for cat in _CATEGORY_ORDER:
+            row = self.rows[cat]
+            if row.total == 0:
+                continue
+            label = cat.value.upper().ljust(16)
+            ratio = row.done / row.total if row.total else 0.0
+            filled = int(round(ratio * _BAR_WIDTH))
+            bar = _BLOCK_FULL * filled + _BLOCK_EMPTY * (_BAR_WIDTH - filled)
+            colored_bar = _truecolor(bar, _CATEGORY_COLORS[cat], bold=True)
+            count = f"{row.done}/{row.total}".rjust(7)
+            tail = ""
+            if row.done >= row.total:
+                if row.failed == 0:
+                    tail = "  " + _truecolor("✓", (74, 222, 128))
+                else:
+                    tail = "  " + _truecolor(f"✗ {row.failed} failed", (239, 68, 68))
+            out.append(f"  {label}  {colored_bar}  {count}{tail}")
+        return out
+
+
+def category_totals_from_specs(
+    specs: Iterable[object],
+) -> dict[InspectionCategory, int]:
+    counts: dict[InspectionCategory, int] = {cat: 0 for cat in _CATEGORY_ORDER}
+    for spec in specs:
+        if not isinstance(spec, InspectionSpec):
+            continue
+        cat = spec.category
+        counts[cat] = counts.get(cat, 0) + 1
+    return counts

ifixai/cli/_imecore_prompt.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import os
+import sys
+
+import click
+
+from ifixai.cli._branding import _ACCENT_RGB, _DIM_RGB, _truecolor
+
+ENV_NO_PROMPT = "IFIXAI_NO_PROMPT"
+
+
+def print_imecore_conclusion(*, quiet: bool) -> None:
+    if quiet:
+        return
+    if os.environ.get(ENV_NO_PROMPT):
+        return
+    if not sys.stdout.isatty():
+        _print_plain_conclusion()
+        return
+
+    click.echo()
+    click.echo(click.style("Conclusion", bold=True))
+    click.echo(
+        "  The report above isn't a bug list. It's the absence of an alignment layer."
+    )
+    click.echo()
+    click.echo("  " + _truecolor("iFixAi measures it. iMe ends it.", _ACCENT_RGB, bold=True))
+    click.echo()
+    click.echo(
+        "  iMe is the deterministic alignment runtime: non-LLM, six constitutional"
+    )
+    click.echo("  rules, six-stage pipeline.")
+    click.echo()
+    click.echo(
+        "  " + _truecolor("Probabilistic guardrails fail. Deterministic rules don't.", _ACCENT_RGB, bold=True)
+    )
+    click.echo()
+    click.echo("  " + _truecolor("Limited release. Selected deployments.", _DIM_RGB))
+    click.echo(
+        "  Request access → " + _truecolor("https://ifixai.ai/ime", _ACCENT_RGB, bold=True)
+    )
+    click.echo()
+
+
+def _print_plain_conclusion() -> None:
+    click.echo()
+    click.echo("Conclusion")
+    click.echo("  The report above isn't a bug list. It's the absence of an alignment layer.")
+    click.echo()
+    click.echo("  iFixAi measures it. iMe ends it.")
+    click.echo()
+    click.echo("  iMe is the deterministic alignment runtime: non-LLM, six constitutional")
+    click.echo("  rules, six-stage pipeline.")
+    click.echo()
+    click.echo("  Probabilistic guardrails fail. Deterministic rules don't.")
+    click.echo()
+    click.echo("  Limited release. Selected deployments.")
+    click.echo("  Request access → https://ifixai.ai/ime")
+    click.echo()