fluiq 0.0.1a1__tar.gz

fluiq-0.0.1a1/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: fluiq
+Version: 0.0.1a1
+Summary: Fluiq SDK
+Requires-Python: >=3.9
+Requires-Dist: requests>=2.31

fluiq-0.0.1a1/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fluiq"
+version = "0.0.1a1"
+description = "Fluiq SDK"
+requires-python = ">=3.9"
+dependencies = [
+    "requests>=2.31",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]

fluiq-0.0.1a1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fluiq-0.0.1a1/src/fluiq/__init__.py

@@ -0,0 +1,2 @@
+from fluiq.config import init as instrument
+from fluiq.decorator import trace

fluiq-0.0.1a1/src/fluiq/client.py

@@ -0,0 +1,20 @@
+import requests
+from fluiq.config import _config
+
+def send_event(data):
+
+    if not _config["enabled"]:
+        return
+    
+    try:
+        r = requests.post(
+            f"{_config['endpoint']}/{_config['version']}/ingest",
+            json={
+                "api_key": _config["api_key"],
+                "event":data
+            },
+            timeout=1
+        )
+        r.raise_for_status()
+    except Exception as e:
+        print("[fluiq] send event failed: ",repr(e))

fluiq-0.0.1a1/src/fluiq/config.py

@@ -0,0 +1,21 @@
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+ENDPOINT = os.getenv("FLUIQ_API_ENDPOINT","http://localhost:8000/api")
+
+_config={
+    "api_key": None,
+    "enabled": True,
+    "version": "v1",
+    "endpoint": ENDPOINT
+}
+
+def init(api_key: str, version="v1", endpoint=ENDPOINT):
+    _config["api_key"] = api_key
+    _config["version"] = version
+    _config["endpoint"] = endpoint
+
+    from fluiq.integrations import init as _integration_init_
+    _integration_init_()

fluiq-0.0.1a1/src/fluiq/decorator/__init__.py

@@ -0,0 +1,67 @@
+import asyncio
+import time
+import uuid
+from fluiq.tracer import log_trace
+from fluiq.integrations.shared.models import LogTrace, TraceType
+from fluiq.integrations.shared.context import (
+    current_parent_id,
+    push_trace_id,
+    pop_trace_id,
+)
+
+
+def _emit(trace_id, parent_id, func_name, args, kwargs, result, exc, start, end):
+    success = exc is None
+    payload = LogTrace(
+        trace_id=trace_id,
+        parent_id=parent_id,
+        integration=TraceType.General_Function,
+        function=func_name,
+        input=str(args) + str(kwargs),
+        output=str(result) if success else str(exc),
+        latency=end - start,
+        success=success,
+    )
+    log_trace(payload.model_dump(mode="json"))
+
+
+def trace(func):
+    if asyncio.iscoroutinefunction(func):
+        async def async_wrapper(*args, **kwargs):
+            trace_id = str(uuid.uuid4())
+            parent_id = current_parent_id()
+            token = push_trace_id(trace_id)
+            start = time.time()
+            exc = None
+            result = None
+            try:
+                result = await func(*args, **kwargs)
+            except Exception as e:
+                exc = e
+            end = time.time()
+            pop_trace_id(token)
+            _emit(trace_id, parent_id, func.__name__, args, kwargs, result, exc, start, end)
+            if exc is not None:
+                raise exc
+            return result
+        return async_wrapper
+
+    def wrapper(*args, **kwargs):
+        trace_id = str(uuid.uuid4())
+        parent_id = current_parent_id()
+        token = push_trace_id(trace_id)
+        start = time.time()
+        exc = None
+        result = None
+        try:
+            result = func(*args, **kwargs)
+        except Exception as e:
+            exc = e
+        end = time.time()
+        pop_trace_id(token)
+        _emit(trace_id, parent_id, func.__name__, args, kwargs, result, exc, start, end)
+        if exc is not None:
+            raise exc
+        return result
+
+    return wrapper

fluiq-0.0.1a1/src/fluiq/evaluations/__init__.py

@@ -0,0 +1,21 @@
+from fluiq.evaluations.base import BaseEvaluator, EvalResult, LLMJudge
+from fluiq.evaluations.hallucination import HallucinationEvaluator
+from fluiq.evaluations.ragas import (
+    AnswerRelevancy,
+    ContextPrecision,
+    ContextRecall,
+    Faithfulness,
+    Ragas,
+)
+
+__all__ = [
+    "BaseEvaluator",
+    "EvalResult",
+    "LLMJudge",
+    "HallucinationEvaluator",
+    "Faithfulness",
+    "AnswerRelevancy",
+    "ContextPrecision",
+    "ContextRecall",
+    "Ragas",
+]

fluiq-0.0.1a1/src/fluiq/evaluations/base.py

@@ -0,0 +1,85 @@
+import json
+import re
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class EvalResult(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    name: str
+    score: float = Field(ge=0.0, le=1.0)
+    passed: Optional[bool] = None
+    reason: Optional[str] = None
+    details: Optional[Dict[str, Any]] = None
+
+
+class BaseEvaluator(ABC):
+    name: str = "evaluator"
+
+    def __init__(self, threshold: float = 0.5):
+        self.threshold = threshold
+
+    @abstractmethod
+    def evaluate(self, **kwargs: Any) -> EvalResult:
+        ...
+
+    def _result(
+        self,
+        score: float,
+        reason: Optional[str] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ) -> EvalResult:
+        return EvalResult(
+            name=self.name,
+            score=float(score),
+            passed=float(score) >= self.threshold,
+            reason=reason,
+            details=details,
+        )
+
+
+def _parse_json_object(raw: str) -> Dict[str, Any]:
+    if not raw:
+        return {}
+    raw = raw.strip()
+    try:
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else {"value": data}
+    except json.JSONDecodeError:
+        pass
+    match = re.search(r"\{.*\}", raw, re.DOTALL)
+    if match:
+        try:
+            data = json.loads(match.group(0))
+            return data if isinstance(data, dict) else {"value": data}
+        except json.JSONDecodeError:
+            return {}
+    return {}
+
+
+def _coerce_contexts(contexts: Any) -> List[str]:
+    if contexts is None:
+        return []
+    if isinstance(contexts, str):
+        return [contexts]
+    if isinstance(contexts, (list, tuple)):
+        return [str(c) for c in contexts if c is not None]
+    return [str(contexts)]
+
+
+def _clamp_unit(value: Any, default: float = 0.0) -> float:
+    try:
+        f = float(value)
+    except (TypeError, ValueError):
+        return default
+    if f < 0.0:
+        return 0.0
+    if f > 1.0:
+        return 1.0
+    return f
+
+
+from fluiq.evaluations.judge import JudgeFn, LLMJudge  # noqa: E402,F401

fluiq-0.0.1a1/src/fluiq/evaluations/hallucination.py

@@ -0,0 +1,142 @@
+from typing import Any, List, Optional
+
+from fluiq.evaluations.base import (
+    BaseEvaluator,
+    EvalResult,
+    LLMJudge,
+    _clamp_unit,
+    _coerce_contexts,
+)
+
+
+_CLAIMS_PROMPT = (
+    "Extract every standalone factual claim from the ANSWER below. "
+    "Return JSON: {{\"claims\": [\"claim 1\", \"claim 2\", ...]}}.\n\n"
+    "ANSWER:\n{answer}"
+)
+
+
+_VERIFY_PROMPT = (
+    "You are checking whether each CLAIM is supported by the REFERENCE. "
+    "A claim is SUPPORTED only if the reference entails it; if the reference "
+    "neither states nor implies it, mark it UNSUPPORTED. Speculation, added "
+    "details, and contradictions are UNSUPPORTED.\n\n"
+    "REFERENCE:\n{reference}\n\n"
+    "CLAIMS:\n{claims}\n\n"
+    "Return JSON: {{\"verdicts\": [{{\"claim\": str, \"supported\": bool, "
+    "\"reason\": str}}]}}"
+)
+
+
+class HallucinationEvaluator(BaseEvaluator):
+    """Hallucination detector.
+
+    Score is 1.0 when no hallucinated claims are detected and 0.0 when every
+    claim in the answer is unsupported. Lower scores are worse. The evaluator
+    accepts either retrieved `contexts` (RAG style) or a single `reference`
+    string (closed-book / ground truth) as the source of truth.
+    """
+
+    name = "hallucination"
+
+    def __init__(
+        self,
+        judge: Optional[LLMJudge] = None,
+        threshold: float = 0.7,
+    ):
+        super().__init__(threshold=threshold)
+        self.judge = judge or LLMJudge()
+
+    def evaluate(
+        self,
+        answer: str,
+        contexts: Any = None,
+        reference: Optional[str] = None,
+        **_: Any,
+    ) -> EvalResult:
+        if not answer or not answer.strip():
+            return self._result(
+                score=1.0,
+                reason="empty answer; nothing to hallucinate",
+                details={"claims": [], "verdicts": []},
+            )
+
+        reference_text = _build_reference(contexts, reference)
+        if not reference_text:
+            return self._result(
+                score=0.0,
+                reason="no reference or contexts provided",
+                details={"claims": [], "verdicts": []},
+            )
+
+        claims = self._extract_claims(answer)
+        if not claims:
+            return self._result(
+                score=1.0,
+                reason="no factual claims extracted from answer",
+                details={"claims": [], "verdicts": []},
+            )
+
+        verdicts = self._verify_claims(reference_text, claims)
+        if not verdicts:
+            return self._result(
+                score=0.0,
+                reason="judge returned no verdicts",
+                details={"claims": claims, "verdicts": []},
+            )
+
+        supported = sum(1 for v in verdicts if v.get("supported") is True)
+        score = _clamp_unit(supported / len(verdicts))
+        unsupported = [v for v in verdicts if v.get("supported") is not True]
+        reason = (
+            f"{supported}/{len(verdicts)} claims supported"
+            if not unsupported
+            else f"{len(unsupported)} unsupported claim(s) of {len(verdicts)}"
+        )
+        return self._result(
+            score=score,
+            reason=reason,
+            details={
+                "claims": claims,
+                "verdicts": verdicts,
+                "supported_count": supported,
+                "total_claims": len(verdicts),
+            },
+        )
+
+    def _extract_claims(self, answer: str) -> List[str]:
+        data = self.judge.judge_json(_CLAIMS_PROMPT.format(answer=answer))
+        raw = data.get("claims") or []
+        if not isinstance(raw, list):
+            return []
+        return [str(c).strip() for c in raw if str(c).strip()]
+
+    def _verify_claims(self, reference: str, claims: List[str]) -> List[dict]:
+        prompt = _VERIFY_PROMPT.format(
+            reference=reference,
+            claims="\n".join(f"- {c}" for c in claims),
+        )
+        data = self.judge.judge_json(prompt)
+        raw = data.get("verdicts") or []
+        if not isinstance(raw, list):
+            return []
+        verdicts: List[dict] = []
+        for v in raw:
+            if not isinstance(v, dict):
+                continue
+            verdicts.append({
+                "claim": str(v.get("claim", "")).strip(),
+                "supported": bool(v.get("supported")),
+                "reason": str(v.get("reason", "")).strip() or None,
+            })
+        return verdicts
+
+
+def _build_reference(contexts: Any, reference: Optional[str]) -> str:
+    parts: List[str] = []
+    for c in _coerce_contexts(contexts):
+        if c.strip():
+            parts.append(c.strip())
+    if reference and reference.strip():
+        parts.append(reference.strip())
+    return "\n\n".join(parts)

fluiq-0.0.1a1/src/fluiq/evaluations/judge.py

@@ -0,0 +1,167 @@
+import os
+from typing import Any, Callable, Dict, Optional
+
+import requests
+
+from fluiq.evaluations.base import _parse_json_object
+
+
+JudgeFn = Callable[[str], str]
+
+PROVIDERS = ("openai", "anthropic", "gemini", "fluiq")
+
+DEFAULT_MODELS: Dict[str, str] = {
+    "openai": "gpt-4o-mini",
+    "anthropic": "claude-3-5-haiku-latest",
+    "gemini": "gemini-2.5-flash",
+    "fluiq": "fluiq-judge",
+}
+
+_SYSTEM_PROMPT = (
+    "You are a strict evaluator. Always respond with a single valid JSON "
+    "object and nothing else."
+)
+
+
+class LLMJudge:
+    """LLM-as-judge with pluggable providers.
+
+    `provider` selects the backend: ``openai`` (default), ``anthropic``,
+    ``gemini``, or ``fluiq``. The first three call the respective vendor
+    SDK directly using the user's own API key. ``fluiq`` proxies the call
+    through the Fluiq API so judging is billed and metered server-side.
+
+    Pass ``judge_fn(prompt: str) -> str`` to short-circuit the provider
+    path entirely (useful for offline testing and self-hosted models).
+    """
+
+    def __init__(
+        self,
+        provider: str = "openai",
+        model: Optional[str] = None,
+        judge_fn: Optional[JudgeFn] = None,
+        api_key: Optional[str] = None,
+        temperature: float = 0.0,
+    ):
+        if provider not in PROVIDERS:
+            raise ValueError(
+                f"Unsupported judge provider: {provider!r}. "
+                f"Use one of: {PROVIDERS}"
+            )
+        self.provider = provider
+        self.model = model or DEFAULT_MODELS[provider]
+        self.temperature = temperature
+        self._judge_fn = judge_fn
+        self._api_key = api_key
+        self._client = None
+
+    def __call__(self, prompt: str) -> str:
+        if self._judge_fn is not None:
+            return self._judge_fn(prompt)
+        if self.provider == "openai":
+            return self._call_openai(prompt)
+        if self.provider == "anthropic":
+            return self._call_anthropic(prompt)
+        if self.provider == "gemini":
+            return self._call_gemini(prompt)
+        if self.provider == "fluiq":
+            return self._call_fluiq(prompt)
+        raise RuntimeError(f"Unsupported provider: {self.provider}")
+
+    def judge_json(self, prompt: str) -> Dict[str, Any]:
+        return _parse_json_object(self(prompt))
+
+    def _call_openai(self, prompt: str) -> str:
+        try:
+            from openai import OpenAI
+        except ImportError as exc:
+            raise RuntimeError(
+                "judge provider 'openai' requires the `openai` package"
+            ) from exc
+        if self._client is None:
+            key = self._api_key or os.getenv("OPENAI_API_KEY")
+            self._client = OpenAI(api_key=key) if key else OpenAI()
+        resp = self._client.chat.completions.create(
+            model=self.model,
+            temperature=self.temperature,
+            messages=[
+                {"role": "system", "content": _SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+            response_format={"type": "json_object"},
+        )
+        return resp.choices[0].message.content or "{}"
+
+    def _call_anthropic(self, prompt: str) -> str:
+        try:
+            import anthropic
+        except ImportError as exc:
+            raise RuntimeError(
+                "judge provider 'anthropic' requires the `anthropic` package"
+            ) from exc
+        if self._client is None:
+            key = self._api_key or os.getenv("ANTHROPIC_API_KEY")
+            self._client = anthropic.Anthropic(api_key=key) if key else anthropic.Anthropic()
+        resp = self._client.messages.create(
+            model=self.model,
+            max_tokens=1024,
+            temperature=self.temperature,
+            system=_SYSTEM_PROMPT,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        for block in getattr(resp, "content", []) or []:
+            text = getattr(block, "text", None)
+            if text:
+                return text
+        return "{}"
+
+    def _call_gemini(self, prompt: str) -> str:
+        try:
+            from google import genai
+            from google.genai import types
+        except ImportError as exc:
+            raise RuntimeError(
+                "judge provider 'gemini' requires the `google-genai` package"
+            ) from exc
+        if self._client is None:
+            key = self._api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
+            self._client = genai.Client(api_key=key) if key else genai.Client()
+        resp = self._client.models.generate_content(
+            model=self.model,
+            contents=prompt,
+            config=types.GenerateContentConfig(
+                system_instruction=_SYSTEM_PROMPT,
+                temperature=self.temperature,
+                response_mime_type="application/json",
+            ),
+        )
+        return getattr(resp, "text", None) or "{}"
+
+    def _call_fluiq(self, prompt: str) -> str:
+        from fluiq.config import _config
+
+        api_key = self._api_key or _config.get("api_key")
+        if not api_key:
+            raise RuntimeError(
+                "judge provider 'fluiq' requires fluiq.instrument(api_key=...) "
+                "or LLMJudge(api_key=...)"
+            )
+        endpoint = _config.get("endpoint")
+        version = _config.get("version", "v1")
+        url = f"{endpoint}/{version}/judge"
+        resp = requests.post(
+            url,
+            json={
+                "api_key": api_key,
+                "model": self.model,
+                "prompt": prompt,
+                "temperature": self.temperature,
+            },
+            timeout=60,
+        )
+        resp.raise_for_status()
+        body = resp.json() or {}
+        return body.get("content") or "{}"
+
+
+