nirizan 0.1.0__py3-none-any.whl

nirizan/__init__.py

@@ -0,0 +1,18 @@
+# src/nirizan/__init__.py
+"""NiriZan: Continuous evaluation infrastructure for production AI systems."""
+from nirizan._logging import (
+    disable_logging,
+    enable_logging,
+    get_logger,
+    set_log_level,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "disable_logging",
+    "enable_logging",
+    "get_logger",
+    "set_log_level",
+]

nirizan/_logging.py

@@ -0,0 +1,113 @@
+# src/nirizan/_logging.py
+from __future__ import annotations
+
+import logging
+import os
+import sys
+from typing import TextIO, Union
+
+_ROOT_LOGGER_NAME = "nirizan"
+_ENV_VAR_LOG_LEVEL = "NIRIZAN_LOG_LEVEL"
+
+LogLevel = Union[int, str]
+
+
+class _NiriZanStreamHandler(logging.StreamHandler[TextIO]):
+    """Internal StreamHandler subclass used to identify NiriZan-managed handlers."""
+
+
+# Library-safe default: silence by default, let the host app opt in.
+logging.getLogger(_ROOT_LOGGER_NAME).addHandler(logging.NullHandler())
+
+
+def get_logger(module_name: str) -> logging.Logger:
+    """Return a logger scoped under the `nirizan` hierarchy.
+
+    Call as `get_logger(__name__)` from any module inside `nirizan/`.
+    """
+    return logging.getLogger(module_name)
+
+
+class NiriZanFormatter(logging.Formatter):
+    """Formats a record with millisecond precision and full exception trace support:
+
+    [INFO] 2026-08-07 14:40:23.279 [NiriZan] tracer.py:182 Started trace 4baf5d17
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        ts = self.formatTime(record, "%Y-%m-%d %H:%M:%S")
+        ts = f"{ts}.{int(record.msecs):03d}"
+
+        msg = (
+            f"[{record.levelname}] {ts} [NiriZan] "
+            f"{record.filename}:{record.lineno} {record.getMessage()}"
+        )
+
+        if record.exc_info:
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+            msg = f"{msg}\n{record.exc_text}"
+
+        if record.stack_info:
+            msg = f"{msg}\n{self.formatStack(record.stack_info)}"
+
+        return msg
+
+
+def _parse_level(level: LogLevel | None) -> int:
+    """Parse int, string, or environment variable into a valid logging level."""
+    if level is None:
+        env_val = os.getenv(_ENV_VAR_LOG_LEVEL, "INFO").upper()
+        return getattr(logging, env_val, logging.INFO)
+
+    if isinstance(level, str):
+        numeric_level = getattr(logging, level.upper(), None)
+        if not isinstance(numeric_level, int):
+            raise ValueError(f"Invalid log level string: {level!r}")
+        return numeric_level
+
+    return level
+
+
+def enable_logging(
+    level: LogLevel | None = None,
+    stream: TextIO | None = None,
+) -> logging.Logger:
+    """Opt-in logging configuration for notebooks, CLI runs, or host apps.
+
+    Accepts both string ("DEBUG", "INFO") and integer (logging.INFO) levels.
+    If no level is supplied, checks the `NIRIZAN_LOG_LEVEL` environment variable,
+    defaulting to `INFO`.
+
+    Idempotent: replaces only NiriZan-managed handlers without touching
+    handlers attached by host applications.
+    """
+    target_level = _parse_level(level)
+    root = logging.getLogger(_ROOT_LOGGER_NAME)
+    root.setLevel(target_level)
+
+    # Clean up prior NiriZan-managed handlers
+    for existing in list(root.handlers):
+        if isinstance(existing, _NiriZanStreamHandler):
+            root.removeHandler(existing)
+
+    handler = _NiriZanStreamHandler(stream or sys.stderr)
+    handler.setFormatter(NiriZanFormatter())
+    root.addHandler(handler)
+
+    return root
+
+
+def set_log_level(level: LogLevel) -> None:
+    """Dynamically update log level for all NiriZan loggers without re-attaching handlers."""
+    target_level = _parse_level(level)
+    root = logging.getLogger(_ROOT_LOGGER_NAME)
+    root.setLevel(target_level)
+
+
+def disable_logging() -> None:
+    """Remove all NiriZan-managed handlers and silence output."""
+    root = logging.getLogger(_ROOT_LOGGER_NAME)
+    for existing in list(root.handlers):
+        if isinstance(existing, _NiriZanStreamHandler):
+            root.removeHandler(existing)

nirizan/gate/__init__.py

@@ -0,0 +1,18 @@
+"""Deployment-aware CI/CD gating layer.
+
+Evaluates regression verdicts alongside statistical confidence intervals to produce
+release deployment signals.
+"""
+# src/nirizan/gate/__init__.py
+
+from nirizan.gate.verdict import (
+    GateVerdict,
+    evaluate_gate,
+    select_decision_metric,
+)
+
+__all__ = [
+    "GateVerdict",
+    "evaluate_gate",
+    "select_decision_metric",
+]

nirizan/gate/ci.py

@@ -0,0 +1,80 @@
+# src/nirizan/gate/ci.py
+from __future__ import annotations
+
+import json
+from typing import TextIO
+
+from nirizan._logging import get_logger
+from nirizan.gate.verdict import GateVerdict
+
+logger = get_logger(__name__)
+
+
+def format_gate_summary(verdict: GateVerdict) -> str:
+    lines = [
+        "| Metric | Severity | P-Value | Effect Size |",
+        "|---|---|---:|---:|",
+    ]
+
+    for regression in verdict.regression_verdicts:
+        p_value = (
+            f"{regression.p_value:.4e}"
+            if regression.p_value is not None
+            else "n/a"
+        )
+
+        effect_size = (
+            f"{regression.effect_size:.3f}"
+            if regression.effect_size is not None
+            else "n/a"
+        )
+
+        lines.append(
+            f"| {regression.metric_name} "
+            f"| {regression.severity.value} "
+            f"| {p_value} "
+            f"| {effect_size} |"
+        )
+
+    lines.append("")
+    lines.append(
+        f"**Gate:** {'PASS' if verdict.passed else 'BLOCK'}"
+    )
+    lines.append(
+        "**95% bootstrap CI:** "
+        f"`{verdict.confidence_interval[0]:.6f}, "
+        f"{verdict.confidence_interval[1]:.6f}`"
+    )
+
+    return "\n".join(lines)
+
+
+def write_github_summary(
+    verdict: GateVerdict,
+    *,
+    output: TextIO,
+) -> None:
+    logger.info(
+        "Writing GitHub CI summary for run_id=%s (passed=%s)",
+        verdict.run_id,
+        verdict.passed,
+    )
+    output.write(format_gate_summary(verdict))
+    output.write("\n")
+
+
+def gate_exit_code(verdict: GateVerdict) -> int:
+    if verdict.passed:
+        logger.info("CI Gate PASSED for run_id=%s", verdict.run_id)
+        return 0
+    else:
+        logger.error("CI Gate BLOCKED for run_id=%s", verdict.run_id)
+        return 1
+
+
+def serialize_gate_verdict(verdict: GateVerdict) -> str:
+    logger.debug("Serializing GateVerdict for run_id=%s", verdict.run_id)
+    return json.dumps(
+        verdict.model_dump(mode="json"),
+        indent=2,
+    )

nirizan/gate/verdict.py

@@ -0,0 +1,170 @@
+# src/nirizan/gate/verdict.py
+from __future__ import annotations
+
+from uuid import UUID
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict, Field
+
+from nirizan._logging import get_logger
+from nirizan.regression.comparator import (
+    RegressionSeverity,
+    RegressionVerdict,
+)
+
+logger = get_logger(__name__)
+
+
+class GateVerdict(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    passed: bool
+    confidence_interval: tuple[float, float]
+    regression_verdicts: list[RegressionVerdict] = Field(
+        default_factory=list
+    )
+    run_id: UUID
+
+
+SEVERITY_WEIGHT = {
+    RegressionSeverity.BLOCKING: 3,
+    RegressionSeverity.WARNING: 2,
+    RegressionSeverity.NONE: 1,
+}
+
+
+def bootstrap_delta_ci(
+    candidate: np.ndarray,
+    baseline: np.ndarray,
+    *,
+    n_bootstrap: int = 5000,
+    confidence: float = 0.95,
+    seed: int = 42,
+) -> tuple[float, float]:
+    if candidate.size == 0 or baseline.size == 0:
+        raise ValueError(
+            "Both distributions must contain observations."
+        )
+
+    if n_bootstrap < 1:
+        raise ValueError("n_bootstrap must be positive.")
+
+    if not 0.0 < confidence < 1.0:
+        raise ValueError(
+            "confidence must be between 0 and 1."
+        )
+
+    logger.debug(
+        "Computing bootstrap delta CI: n_bootstrap=%d, confidence=%.2f, candidate_n=%d, baseline_n=%d",
+        n_bootstrap,
+        confidence,
+        candidate.size,
+        baseline.size,
+    )
+
+    rng = np.random.default_rng(seed)
+
+    candidate_samples = rng.choice(
+        candidate,
+        size=(n_bootstrap, candidate.size),
+        replace=True,
+    )
+
+    baseline_samples = rng.choice(
+        baseline,
+        size=(n_bootstrap, baseline.size),
+        replace=True,
+    )
+
+    deltas = (
+        candidate_samples.mean(axis=1)
+        - baseline_samples.mean(axis=1)
+    )
+
+    alpha = 1.0 - confidence
+
+    ci = (
+        float(np.quantile(deltas, alpha / 2.0)),
+        float(np.quantile(deltas, 1.0 - alpha / 2.0)),
+    )
+    logger.debug("Bootstrap CI computed: [%.6f, %.6f]", ci[0], ci[1])
+    return ci
+
+
+def select_decision_metric(
+    verdicts: list[RegressionVerdict],
+) -> RegressionVerdict:
+    if not verdicts:
+        raise ValueError("At least one verdict is required.")
+
+    selected = min(
+        verdicts,
+        key=lambda verdict: (
+            -SEVERITY_WEIGHT[verdict.severity],
+            (
+                verdict.effect_size
+                if verdict.effect_size is not None
+                else 0.0
+            ),
+        ),
+    )
+    logger.debug(
+        "Selected decision metric '%s' (severity=%s, effect_size=%s)",
+        selected.metric_name,
+        selected.severity.value,
+        selected.effect_size,
+    )
+    return selected
+
+
+def evaluate_gate(
+    *,
+    verdicts: list[RegressionVerdict],
+    scores_by_metric: dict[
+        str,
+        tuple[np.ndarray, np.ndarray],
+    ],
+) -> GateVerdict:
+    if not verdicts:
+        raise ValueError(
+            "Gate requires at least one regression verdict."
+        )
+
+    logger.info("Evaluating gate across %d regression verdict(s)", len(verdicts))
+
+    decision_metric = select_decision_metric(verdicts)
+
+    candidate_scores, baseline_scores = scores_by_metric[
+        decision_metric.metric_name
+    ]
+
+    confidence_interval = bootstrap_delta_ci(
+        candidate_scores,
+        baseline_scores,
+    )
+
+    blocking_verdicts = [
+        verdict
+        for verdict in verdicts
+        if verdict.severity == RegressionSeverity.BLOCKING
+    ]
+    passed = len(blocking_verdicts) == 0
+
+    if passed:
+        logger.info(
+            "Gate evaluation result: PASSED for run_id=%s",
+            decision_metric.run_id,
+        )
+    else:
+        logger.warning(
+            "Gate evaluation result: BLOCKED for run_id=%s (%d blocking regression(s))",
+            decision_metric.run_id,
+            len(blocking_verdicts),
+        )
+
+    return GateVerdict(
+        passed=passed,
+        confidence_interval=confidence_interval,
+        regression_verdicts=verdicts,
+        run_id=decision_metric.run_id,
+    )

nirizan/instrumentation/__init__.py

@@ -0,0 +1,6 @@
+# src/nirizan/instrumentation/__init__.py
+"""Instrumentation layer for capturing OpenTelemetry-style spans and traces.
+
+Provides atomic tracing primitives, decorators, SDK context managers,
+and asynchronous trace exporters.
+"""

nirizan/instrumentation/exporters.py

@@ -0,0 +1,47 @@
+from abc import ABC, abstractmethod
+import logging
+
+from nirizan.instrumentation.spans import Trace
+
+logger = logging.getLogger(__name__)
+
+
+class BaseExporter(ABC):
+    """Abstract base class for all trace exporters."""
+
+    @abstractmethod
+    async def export(self, trace: Trace) -> None:
+        """Export a completed Trace to storage or a remote collector."""
+        pass
+
+    async def shutdown(self) -> None:
+        """Release underlying connections or background workers."""
+        pass
+
+
+class InMemoryExporter(BaseExporter):
+    """In-memory trace collector designed for unit tests and local experiments."""
+
+    def __init__(self) -> None:
+        self._traces: list[Trace] = []
+
+    async def export(self, trace: Trace) -> None:
+        self._traces.append(trace)
+
+    def get_traces(self) -> list[Trace]:
+        return list(self._traces)
+
+    def clear(self) -> None:
+        self._traces.clear()
+
+
+class ConsoleExporter(BaseExporter):
+    """Logs trace telemetry directly to standard logger output."""
+
+    async def export(self, trace: Trace) -> None:
+        logger.info(
+            "Trace Exported | ID: %s | App: %s | Spans: %d",
+            trace.trace_id,
+            trace.application_name,
+            len(trace.spans),
+        )

nirizan/instrumentation/sdk.py

@@ -0,0 +1,117 @@
+# src/nirizan/instrumentation/sdk.py
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable, Coroutine, Optional, ParamSpec, TypeVar
+from uuid import UUID
+
+from nirizan._logging import get_logger
+from nirizan.instrumentation.exporters import BaseExporter
+from nirizan.instrumentation.spans import SpanKind
+from nirizan.instrumentation.tracer import Tracer
+
+logger = get_logger(__name__)
+
+_GLOBAL_TRACER: Optional[Tracer] = None
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def init_tracer(
+    application_name: str, exporter: Optional[BaseExporter] = None
+) -> Tracer:
+    """Initialize and register the global tracer instance."""
+    global _GLOBAL_TRACER
+    tracer = Tracer(application_name=application_name, exporter=exporter)
+    _GLOBAL_TRACER = tracer
+    logger.info("Initialized global tracer for application '%s'", application_name)
+    return tracer
+
+
+def get_tracer() -> Optional[Tracer]:
+    """Return the currently configured global tracer instance."""
+    return _GLOBAL_TRACER
+
+
+def start_session(session_id: Optional[UUID] = None) -> Any:
+    """SDK-level context manager pass-through for grouping traces into a session."""
+    tracer = get_tracer()
+    if tracer is None:
+        raise RuntimeError("Tracer is not initialized. Call init_tracer() first.")
+    logger.debug("Starting SDK session (session_id=%s)", session_id)
+    return tracer.session(session_id=session_id)
+
+
+def _format_input_payload(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str | None:
+    if args:
+        return str(args[0])
+    if kwargs:
+        return str(next(iter(kwargs.values())))
+    return None
+
+
+def trace_span(
+    kind: SpanKind,
+    name: Optional[str] = None,
+    tracer: Optional[Tracer] = None,
+) -> Callable[[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any, R]]]:
+    """Decorator to instrument an async function with full signature preservation."""
+    def decorator(
+        func: Callable[P, Coroutine[Any, Any, R]]
+    ) -> Callable[P, Coroutine[Any, Any, R]]:
+        span_name = name or func.__name__
+
+        @functools.wraps(func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+            active_tracer = tracer or get_tracer()
+            if active_tracer is None:
+                raise RuntimeError(
+                    "Tracer is not initialized. Call init_tracer() before executing traced code."
+                )
+
+            input_payload = _format_input_payload(args, kwargs)
+
+            async with active_tracer.start_span(
+                name=span_name, kind=kind, input_payload=input_payload
+            ) as handle:
+                result = await func(*args, **kwargs)
+                if result is not None and handle.output_payload is None:
+                    handle.output_payload = str(result)
+                return result
+
+        return wrapper
+
+    return decorator
+
+
+def planning(
+    name: Optional[str] = None,
+    tracer: Optional[Tracer] = None,
+) -> Callable[[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any, R]]]:
+    """Convenience decorator for PLANNING span instrumentation."""
+    return trace_span(kind=SpanKind.PLANNING, name=name, tracer=tracer)
+
+
+def retrieval(
+    name: Optional[str] = None,
+    tracer: Optional[Tracer] = None,
+) -> Callable[[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any, R]]]:
+    """Convenience decorator for RETRIEVAL span instrumentation."""
+    return trace_span(kind=SpanKind.RETRIEVAL, name=name, tracer=tracer)
+
+
+def generation(
+    name: Optional[str] = None,
+    tracer: Optional[Tracer] = None,
+) -> Callable[[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any, R]]]:
+    """Convenience decorator for GENERATION span instrumentation."""
+    return trace_span(kind=SpanKind.GENERATION, name=name, tracer=tracer)
+
+
+def tool_use(
+    name: Optional[str] = None,
+    tracer: Optional[Tracer] = None,
+) -> Callable[[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any, R]]]:
+    """Convenience decorator for TOOL_USE span instrumentation."""
+    return trace_span(kind=SpanKind.TOOL_USE, name=name, tracer=tracer)

nirizan/instrumentation/sessions.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class Session(BaseModel):
+    """Groups multiple Trace ids belonging to one multi-turn conversation; not frozen, open until ended."""
+
+    model_config = ConfigDict(strict=True)
+
+    session_id: UUID
+    application_name: str = Field(min_length=1)
+    trace_ids: list[UUID] = Field(default_factory=list)
+    started_at: datetime
+    ended_at: datetime | None = None

nirizan/instrumentation/spans.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class SpanKind(str, Enum):
+    """The functional role of an execution span."""
+
+    PLANNING = "planning"
+    RETRIEVAL = "retrieval"
+    TOOL_USE = "tool_use"
+    GENERATION = "generation"
+
+
+class Span(BaseModel):
+    """The atomic unit of instrumentation: one step in an AI execution graph."""
+
+    model_config = ConfigDict(frozen=True, strict=True)
+
+    span_id: UUID
+    trace_id: UUID
+    parent_span_id: UUID | None = None
+    kind: SpanKind
+    name: str = Field(min_length=1, max_length=200)
+    started_at: datetime
+    ended_at: datetime
+    attributes: dict[str, str | int | float | bool] = Field(default_factory=dict)
+    input_payload: str | None = None
+    output_payload: str | None = None
+
+
+class Trace(BaseModel):
+    """An ordered collection of spans belonging to a single invocation."""
+
+    model_config = ConfigDict(strict=True)
+
+    trace_id: UUID
+    application_name: str = Field(min_length=1)
+    spans: list[Span] = Field(default_factory=list)
+    created_at: datetime
+    code_commit: str | None = None       # Phase 3: stamped by collector.py at ingest
+    data_snapshot_id: str | None = None  # Phase 3: stamped by collector.py at ingest
+    session_id: UUID | None = None       # Phase 3: set when captured inside Tracer.session(...)
+
+    @model_validator(mode="after")
+    def validate_span_trace_ids(self) -> Trace:
+        """Ensure all spans in the trace share the trace's trace_id."""
+        for span in self.spans:
+            if span.trace_id != self.trace_id:
+                raise ValueError(
+                    f"Span {span.span_id} trace_id ({span.trace_id}) "
+                    f"does not match Trace trace_id ({self.trace_id})"
+                )
+        return self
+
+    def spans_of_kind(self, kind: SpanKind) -> list[Span]:
+        """Return all spans matching a specific SpanKind."""
+        return [s for s in self.spans if s.kind == kind]