wisent 0.1.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

wisent/classifiers/models/logistic.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import torch
+from torch import nn
+
+from wisent.classifiers.core.atoms import BaseClassifier
+
+__all__ = ["LogisticClassifier"]
+
+class LogisticModel(nn.Module):
+    """Simple logistic regression model for activation classification."""
+    def __init__(self, input_dim: int):
+        super().__init__()
+        self.linear = nn.Linear(input_dim, 1)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        logits = self.linear(x)
+        if logits.ndim == 1:
+            logits = logits.unsqueeze(1)
+        return self.sigmoid(logits)
+
+
+class LogisticClassifier(BaseClassifier):
+    name = "logistic"
+    description = "One-layer logistic regression over dense features"
+
+    def build_model(self, input_dim: int, **_: object) -> nn.Module:
+        return LogisticModel(input_dim)

wisent/classifiers/models/mlp.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import torch
+from torch import nn
+
+from wisent.classifiers.core.atoms import BaseClassifier
+
+__all__ = ["MLPClassifier"]
+
+class MLPModel(nn.Module):
+    """Multi-layer perceptron for activation classification."""
+    def __init__(self, input_dim: int, hidden_dim: int = 128, dropout: float = 0.2):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(input_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim, hidden_dim // 2),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim // 2, 1),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        out = self.net(x)
+        if out.ndim == 1:
+            out = out.unsqueeze(1)
+        return out
+
+
+class MLPClassifier(BaseClassifier):
+    name = "mlp"
+    description = "Two-layer MLP with dropout and ReLU"
+
+    def __init__(self, *, hidden_dim: int = 128, **base_kwargs):
+        super().__init__(**base_kwargs)
+        self._hidden_dim = int(hidden_dim)
+
+    def build_model(self, input_dim: int, **model_params: object) -> nn.Module:
+        hd = int(model_params.get("hidden_dim", self._hidden_dim))  
+        dp = float(model_params.get("dropout", 0.2))
+        self._hidden_dim = hd
+        return MLPModel(input_dim, hidden_dim=hd, dropout=dp)
+
+    def model_hyperparams(self) -> dict[str, int]:
+        return {"hidden_dim": self._hidden_dim, "dropout": 0.2}

wisent/cli/classifiers/classifier_rotator.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import inspect
+import pkgutil
+from pathlib import Path
+from typing import Any
+
+from wisent.core.classifiers.core.atoms import BaseClassifier, ClassifierError, ClassifierTrainReport
+
+__all__ = ["ClassifierRotator"]
+
+class ClassifierRotator:
+    """
+    Discover, list, and delegate to registered classifiers.
+    """
+
+    def __init__(
+        self,
+        classifier: str | BaseClassifier | type[BaseClassifier] | None = None,
+        classifiers_location: str | Path = "wisent_guard.core.classifiers.models",
+        autoload: bool = True,
+        **classifier_kwargs: Any,
+    ) -> None:
+        if autoload:
+            self.discover_classifiers(classifiers_location)
+        self._classifier = self._resolve_classifier(classifier, **classifier_kwargs)
+
+    @staticmethod
+    def discover_classifiers(location: str | Path = "wisent_guard.core.classifiers.models") -> None:
+        """
+        Import all classifier modules so BaseClassifier subclasses self-register.
+
+        - If `location` is a dotted module path (str without existing FS path),
+          import that package and iterate its __path__ (works with namespace packages).
+        - If `location` is an existing directory (Path/str), import all .py files inside.
+        """
+        loc_path = Path(str(location))
+        if loc_path.exists() and loc_path.is_dir():
+            ClassifierRotator._import_all_py_in_dir(loc_path)
+            return
+
+        if not isinstance(location, str):
+            raise ClassifierError(
+                f"Invalid classifiers location: {location!r}. Provide a dotted module path or a directory."
+            )
+
+        try:
+            pkg = importlib.import_module(location)
+        except ModuleNotFoundError as exc:
+            raise ClassifierError(
+                f"Cannot import classifier package {location!r}. "
+                f"Use a dotted path (no leading slash) and ensure your project root is on PYTHONPATH."
+            ) from exc
+
+        search_paths = list(getattr(pkg, "__path__", []))
+        if not search_paths:
+            pkg_file = getattr(pkg, "__file__", None)
+            if pkg_file:
+                search_paths = [str(Path(pkg_file).parent)]
+
+        for _finder, name, _ispkg in pkgutil.iter_modules(search_paths):
+            if name.startswith("_"):
+                continue
+            importlib.import_module(f"{location}.{name}")
+
+    @staticmethod
+    def _import_all_py_in_dir(directory: Path) -> None:
+        for py in directory.glob("*.py"):
+            if py.name.startswith("_"):
+                continue
+            mod_name = f"_dyn_classifiers_{py.stem}"
+            spec = importlib.util.spec_from_file_location(mod_name, py)
+            if spec and spec.loader:
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)  # type: ignore[attr-defined]
+
+    @staticmethod
+    def list_classifiers() -> list[dict[str, Any]]:
+        out: list[dict[str, Any]] = []
+        for name, cls in BaseClassifier.list_registered().items():
+            out.append(
+                {
+                    "name": name,
+                    "description": getattr(cls, "description", ""),
+                    "class": f"{cls.__module__}.{cls.__name__}",
+                }
+            )
+        return sorted(out, key=lambda x: x["name"])
+
+    @staticmethod
+    def _resolve_classifier(
+        classifier: str | BaseClassifier | type[BaseClassifier] | None,
+        **kwargs: Any,
+    ) -> BaseClassifier:
+        if classifier is None:
+            registry = BaseClassifier.list_registered()
+            if not registry:
+                raise ClassifierError("No classifiers registered.")
+            # Deterministic pick: first by name
+            return next(iter(sorted(registry.items())))[1](**kwargs)
+        if isinstance(classifier, BaseClassifier):
+            return classifier
+        if inspect.isclass(classifier) and issubclass(classifier, BaseClassifier):
+            return classifier(**kwargs)
+        if isinstance(classifier, str):
+            cls = BaseClassifier.get(classifier)
+            return cls(**kwargs)
+        raise TypeError(
+            "classifier must be None, a name (str), BaseClassifier instance, or BaseClassifier subclass."
+        )
+
+
+    def use(self, classifier: str | BaseClassifier | type[BaseClassifier], **kwargs: Any) -> None:
+        self._classifier = self._resolve_classifier(classifier, **kwargs)
+
+    def fit(self, X, y, **kwargs) -> ClassifierTrainReport:
+        return self._classifier.fit(X, y, **kwargs)
+
+    def predict(self, X):
+        return self._classifier.predict(X)
+
+    def predict_proba(self, X):
+        return self._classifier.predict_proba(X)
+
+    def evaluate(self, X, y) -> dict[str, float]:
+        return self._classifier.evaluate(X, y)
+
+    def save_model(self, path: str) -> None:
+        self._classifier.save_model(path)
+
+    def load_model(self, path: str) -> None:
+        self._classifier.load_model(path)
+
+    def set_threshold(self, threshold: float) -> None:
+        self._classifier.set_threshold(threshold)

wisent/cli/cli_logger.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from datetime import datetime, timezone
+from typing import Any, Mapping
+
+__all__ = [
+    "setup_logger",
+    "bind",
+    "JsonFormatter",
+    "ContextAdapter",
+    "add_file_handler",
+]
+
+class JsonFormatter(logging.Formatter):
+    """
+    Minimal JSON formatter with structured fields + extras.
+    """
+    _STD = {
+        "name", "msg", "args", "levelname", "levelno", "pathname",
+        "filename", "module", "exc_info", "exc_text", "stack_info",
+        "lineno", "funcName", "created", "msecs", "relativeCreated",
+        "thread", "threadName", "processName", "process"
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "ts": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "file": record.filename,
+            "func": record.funcName,
+            "line": record.lineno,
+        }
+        extras = {
+            k: v for k, v in record.__dict__.items()
+            if k not in self._STD and not k.startswith("_")
+        }
+        if extras:
+            payload["extra"] = extras
+        if record.exc_info:
+            payload["exc"] = self.formatException(record.exc_info)
+        return json.dumps(payload, ensure_ascii=False)
+
+
+class ContextAdapter(logging.LoggerAdapter):
+    """
+    LoggerAdapter that ensures persistent context fields appear in every log entry.
+    """
+    def process(self, msg, kwargs):
+        extra = kwargs.get("extra", {})
+        extra.update(self.extra or {})
+        kwargs["extra"] = extra
+        return msg, kwargs
+
+
+class _EnsureContextFilter(logging.Filter):
+    """
+    Adds default values for context keys so format strings never KeyError.
+    """
+    def __init__(self, defaults: Mapping[str, Any] | None = None):
+        super().__init__()
+        self.defaults = dict(defaults or {})
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        for k, v in self.defaults.items():
+            if not hasattr(record, k):
+                setattr(record, k, v)
+        return True
+
+
+def setup_logger(
+    name: str = "wisent",
+    level: int = logging.INFO,
+    *,
+    json_logs: bool = False,
+    stream = sys.stderr,
+) -> logging.Logger:
+    """
+    Create or return a named logger with a single stream handler.
+    Safe to call multiple times; won’t duplicate handlers.
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    if not logger.handlers:
+        handler = logging.StreamHandler(stream)
+        if json_logs:
+            handler.setFormatter(JsonFormatter())
+        else:
+            handler.setFormatter(logging.Formatter(
+                fmt="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s "
+                    "[file=%(filename)s func=%(funcName)s line=%(lineno)d] "
+                    "%(task_name)s%(subtask)s",
+                datefmt="%Y-%m-%dT%H:%M:%S%z",
+            ))
+        # ensure context placeholders always exist
+        handler.addFilter(_EnsureContextFilter({"task_name": "", "subtask": ""}))
+        logger.addHandler(handler)
+        logger.propagate = False
+    return logger
+
+
+def add_file_handler(
+    logger: logging.Logger,
+    filepath: str,
+    *,
+    level: int | None = None,
+    json_logs: bool = False,
+) -> None:
+    """
+    Optionally add a file handler (e.g., for long-running CLI jobs).
+    """
+    fh = logging.FileHandler(filepath, encoding="utf-8")
+    fh.setLevel(level or logger.level)
+    if json_logs:
+        fh.setFormatter(JsonFormatter())
+    else:
+        fh.setFormatter(logging.Formatter(
+            fmt="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s "
+                "[file=%(filename)s func=%(funcName)s line=%(lineno)d] "
+                "%(task_name)s%(subtask)s",
+            datefmt="%Y-%m-%dT%H:%M:%S%z",
+        ))
+    fh.addFilter(_EnsureContextFilter({"task_name": "", "subtask": ""}))
+    logger.addHandler(fh)
+
+
+def bind(
+    logger: logging.Logger | ContextAdapter,
+    **extra: Any
+) -> ContextAdapter:
+    """
+    Return a ContextAdapter with merged extras.
+    Works whether you pass a raw Logger or an existing ContextAdapter.
+    """
+    if isinstance(logger, ContextAdapter):
+        merged = {**logger.extra, **extra}
+        return ContextAdapter(logger.logger, merged)
+    return ContextAdapter(logger, extra)

wisent/cli/data_loaders/data_loader_rotator.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import inspect
+import pkgutil
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Type, Union
+
+from wisent.core.data_loaders.core.atoms import BaseDataLoader, DataLoaderError, LoadDataResult
+
+class DataLoaderRotator:
+    """Discover/select a data loader and use it to load data."""
+    def __init__(
+        self,
+        loader: Union[str, BaseDataLoader, Type[BaseDataLoader], None] = None,
+        loaders_location: Union[str, Path] = "wisent_guard.core.data_loaders.loaders",
+        autoload: bool = True,
+        **default_loader_kwargs: Any,
+    ) -> None:
+        self._scope_prefix = (
+            loaders_location if isinstance(loaders_location, str)
+            else Path(loaders_location).as_posix().replace("/", ".")
+        )
+        if autoload:
+            self.discover_loaders(loaders_location)
+        self._loader = self._resolve_loader(loader, **default_loader_kwargs)
+
+    @staticmethod
+    def discover_loaders(location: Union[str, Path]) -> None:
+        loc_path = Path(str(location))
+        if loc_path.exists() and loc_path.is_dir():
+            for py in loc_path.glob("*.py"):
+                if py.name.startswith("_"):
+                    continue
+                mod_name = f"_dyn_dataloaders_{py.stem}"
+                spec = importlib.util.spec_from_file_location(mod_name, py)
+                if spec and spec.loader:
+                    module = importlib.util.module_from_spec(spec)
+                    spec.loader.exec_module(module)  # type: ignore[attr-defined]
+            return
+
+        if not isinstance(location, str):
+            raise DataLoaderError(f"Invalid loaders location: {location!r}. Provide dotted path or a directory.")
+
+        pkg = importlib.import_module(location)
+        search_paths = list(getattr(pkg, "__path__", [])) or [Path(getattr(pkg, "__file__", "")).parent.as_posix()]
+        for _, name, _ in pkgutil.iter_modules(search_paths):
+            if name.startswith("_"):
+                continue
+            importlib.import_module(f"{location}.{name}")
+
+    def _scoped_registry(self) -> dict[str, type[BaseDataLoader]]:
+        reg = BaseDataLoader.list_registered()
+        return {n: c for n, c in reg.items() if c.__module__.startswith(self._scope_prefix)}
+
+    @staticmethod
+    def list_loaders(scope_prefix: Optional[str] = None) -> List[Dict[str, Any]]:
+        reg = BaseDataLoader.list_registered()
+        if scope_prefix:
+            reg = {n: c for n, c in reg.items() if c.__module__.startswith(scope_prefix)}
+        return [
+            {"name": n, "description": getattr(c, "description", ""), "class": f"{c.__module__}.{c.__name__}"}
+            for n, c in sorted(reg.items(), key=lambda kv: kv[0])
+        ]
+
+    def _resolve_loader(
+        self,
+        loader: Union[str, BaseDataLoader, Type[BaseDataLoader], None],
+        **kwargs: Any,
+    ) -> BaseDataLoader:
+        reg = self._scoped_registry()
+        if loader is None:
+            if not reg:
+                raise DataLoaderError(f"No data loaders registered under {self._scope_prefix!r}.")
+            cls = next(iter(sorted(reg.items(), key=lambda kv: kv[0])))[1]
+            return cls(**kwargs)
+        if isinstance(loader, BaseDataLoader):
+            loader.kwargs = {**kwargs, **loader.kwargs}
+            return loader
+        if inspect.isclass(loader) and issubclass(loader, BaseDataLoader):
+            if not loader.__module__.startswith(self._scope_prefix):
+                raise DataLoaderError(f"Loader class must live under {self._scope_prefix!r}.")
+            return loader(**kwargs)
+        if isinstance(loader, str):
+            if loader not in reg:
+                raise DataLoaderError(f"Unknown loader {loader!r} in scope {self._scope_prefix!r}.")
+            return reg[loader](**kwargs)
+        raise TypeError("loader must be None, a name (str), BaseDataLoader instance, or BaseDataLoader subclass.")
+
+    def use(self, loader: Union[str, BaseDataLoader, Type[BaseDataLoader]], **kwargs: Any) -> None:
+        self._loader = self._resolve_loader(loader, **kwargs)
+
+    def load(self, **kwargs: Any) -> LoadDataResult:
+        merged = {**getattr(self._loader, "kwargs", {}), **kwargs}
+        return self._loader.load(**merged)

wisent/cli/evaluators/evaluator_rotator.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import pkgutil
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Union, Type
+import inspect
+import logging
+
+from wisent.core.evaluators.core.atoms import BaseEvaluator, EvalResult, EvaluatorError
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluatorRotator:
+    """Orchestrates evaluator selection and execution with flexible discovery."""
+
+    def __init__(
+        self,
+        evaluator: Union[str, BaseEvaluator, Type[BaseEvaluator], None] = None,
+        task_name: Optional[str] = None,
+        evaluators_location: Union[str, Path] = "wisent_guard.core.evaluators.oracles",
+        autoload: bool = True,
+    ) -> None:
+        if autoload:
+            self.discover_evaluators(evaluators_location)
+        self._evaluator = self._resolve_evaluator(evaluator)
+        self._task_name = task_name
+
+    @staticmethod
+    def discover_evaluators(location: Union[str, Path] = "wisent_guard.core.evaluators.oracles") -> None:
+        """
+        Import all evaluator modules so BaseEvaluator subclasses self-register.
+
+        - If `location` is a dotted module path (str without existing FS path),
+          import that package and iterate its __path__ (works with namespace packages).
+        - If `location` is an existing directory (Path/str), import all .py files inside.
+        """
+
+        loc_path = Path(str(location))
+        if loc_path.exists() and loc_path.is_dir():
+            EvaluatorRotator._import_all_py_in_dir(loc_path)
+            return
+
+        if not isinstance(location, str):
+            raise EvaluatorError(
+                f"Invalid evaluators location: {location!r}. Provide a dotted module path or a directory."
+            )
+
+        try:
+            pkg = importlib.import_module(location)
+        except ModuleNotFoundError as exc:
+            raise EvaluatorError(
+                f"Cannot import evaluator package {location!r}. "
+                f"Use dotted path (no leading slash) and ensure your project root is on PYTHONPATH."
+            ) from exc
+
+        search_paths = list(getattr(pkg, "__path__", []))  # supports namespace pkgs
+        if not search_paths:
+            # Some packages may still have __file__ only
+            pkg_file = getattr(pkg, "__file__", None)
+            if pkg_file:
+                search_paths = [str(Path(pkg_file).parent)]
+
+        for finder, name, ispkg in pkgutil.iter_modules(search_paths):
+            if name.startswith("_"):
+                continue
+            importlib.import_module(f"{location}.{name}")
+
+    @staticmethod
+    def _import_all_py_in_dir(directory: Path) -> None:
+        for py in directory.glob("*.py"):
+            if py.name.startswith("_"):
+                continue
+            mod_name = f"_dyn_evaluators_{py.stem}"
+            spec = importlib.util.spec_from_file_location(mod_name, py)
+            if spec and spec.loader:
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)  # type: ignore[attr-defined]
+
+    @staticmethod
+    def list_evaluators() -> List[Dict[str, Any]]:
+        out: List[Dict[str, Any]] = []
+        for name, cls in BaseEvaluator.list_registered().items():
+            out.append(
+                {
+                    "name": name,
+                    "description": getattr(cls, "description", ""),
+                    "task_names": list(getattr(cls, "task_names", ())),
+                    "class": f"{cls.__module__}.{cls.__name__}",
+                }
+            )
+        return sorted(out, key=lambda x: x["name"])
+
+    @staticmethod
+    def _resolve_evaluator(
+        evaluator: Union[str, BaseEvaluator, Type[BaseEvaluator], None]
+    ) -> BaseEvaluator:
+        if evaluator is None:
+            registry = BaseEvaluator.list_registered()
+            if "lm_eval" in registry:
+                return registry["lm_eval"]()
+            if registry:
+                return next(iter(registry.values()))()
+            raise EvaluatorError("No evaluators registered.")
+        if isinstance(evaluator, BaseEvaluator):
+            return evaluator
+        if inspect.isclass(evaluator) and issubclass(evaluator, BaseEvaluator):
+            return evaluator()
+        if isinstance(evaluator, str):
+            cls = BaseEvaluator.get(evaluator)
+            return cls()
+        raise TypeError(
+            "evaluator must be None, a name (str), BaseEvaluator instance, or BaseEvaluator subclass."
+        )
+
+    def use(self, evaluator: Union[str, BaseEvaluator, Type[BaseEvaluator]]) -> None:
+        self._evaluator = self._resolve_evaluator(evaluator)
+
+    def evaluate(self, response: str, expected: Any, **kwargs) -> EvalResult:
+        kwargs.setdefault("task_name", self._task_name)
+        return self._evaluator.evaluate(response, expected, **kwargs)
+
+    def evaluate_batch(
+        self, responses: Sequence[str], expected_answers: Sequence[Any], **kwargs
+    ) -> List[EvalResult]:
+        kwargs.setdefault("task_name", self._task_name)
+        return self._evaluator.evaluate_batch(responses, expected_answers, **kwargs)
+
+
+if __name__ == "__main__":
+    from evaluator_rotator import EvaluatorRotator
+
+    rot = EvaluatorRotator(
+    evaluators_location="wisent_guard.core.evaluators.oracles",  # << no leading slash
+    autoload=True,
+    )
+
+    rot.list_evaluators()
+    print("Available evaluators:")
+    for ev in rot.list_evaluators():
+        print(f" - {ev['name']}: {ev['description']} (tasks: {', '.join(ev['task_names'])})")
+
+    # rot.use("nlp")
+    # res = rot.evaluate("The answer is probably 42", expected="The answer is 12")
+
+    # print(res)