coatlearn 0.1.0__py3-none-any.whl

coatlearn/__init__.py

@@ -0,0 +1,26 @@
+"""COAT: causal discovery over unstructured data with large language models.
+
+The public surface is intentionally small::
+
+    from coatlearn import ModelConfig, RunConfig, run_mb, run_pag, run_adj
+
+``run_mb`` / ``run_pag`` / ``run_adj`` wrap the core experiment classes
+(``COATMB`` / ``COATPAG`` / ``COATADJ``) without re-implementing any
+algorithm.  They validate inputs first, then resolve model clients, and
+return a strictly JSON-safe result dictionary with a conservative
+``status`` field.
+
+Importing this package has no side effects: no model client is
+constructed, no credentials are read, no experiment object is created,
+no NLTK corpus is downloaded, and no dataset is accessed.  Heavy
+dependencies (causal-learn and friends) are imported only when a ``run_*``
+function is actually called.
+
+Independent model, annotation and causal operations are available from their
+functional subpackages. Importing those interfaces starts no model inference.
+"""
+
+from coatlearn.models.openai import ModelConfig
+from .api import RunConfig, run_mb, run_pag, run_adj
+
+__all__ = ["ModelConfig", "RunConfig", "run_mb", "run_pag", "run_adj"]

coatlearn/__main__.py

@@ -0,0 +1,5 @@
+"""Run the installed command with python -m coatlearn."""
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

coatlearn/annotation/__init__.py

@@ -0,0 +1,12 @@
+"""Measure user-defined factors without running causal discovery."""
+from .types import MeasurementSpec, FactorAnnotation, UnsupportedMeasurement
+from .definitions import factor_definition_to_spec, general_definition_to_spec
+from .measurement import JevAnnotator, KevAnnotator, FactorAnnotator
+from .batch import annotate, AnnotationResult
+from .judge import JevFactorJudge, review_definition
+from .dedup import JevFactorDeduplicator, compare_definitions
+
+__all__ = ["annotate", "AnnotationResult", "MeasurementSpec", "FactorAnnotation",
+           "UnsupportedMeasurement", "factor_definition_to_spec", "general_definition_to_spec",
+           "FactorAnnotator", "JevAnnotator", "KevAnnotator", "JevFactorJudge",
+           "JevFactorDeduplicator", "review_definition", "compare_definitions"]

coatlearn/annotation/batch.py

@@ -0,0 +1,90 @@
+"""Batch measurement of fixed factor definitions, with no discovery or screening."""
+from __future__ import annotations
+
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+import pandas as pd
+
+from .definitions import factor_definition_to_spec
+from .types import MeasurementSpec
+
+
+@dataclass
+class AnnotationResult:
+    """Aligned values, per-measurement metadata and the finalized definitions."""
+
+    values: pd.DataFrame
+    metadata: pd.DataFrame
+    specs: dict[str, MeasurementSpec]
+
+
+def resolve_annotator(model):
+    """Resolve a preset or model configuration after local data validation."""
+    from coatlearn.models.presets import ModelPreset
+    from coatlearn.models.jev import JevConfig
+    from coatlearn.models.systemone import SystemOneClient
+    from .measurement import JevAnnotator, decision_annotator
+
+    if isinstance(model, ModelPreset):
+        model = model._resolve("annotation")
+    if isinstance(model, JevConfig):
+        return decision_annotator(model)
+    if callable(getattr(model, "annotate_record", None)):
+        return model
+    if isinstance(model, SystemOneClient):
+        annotator = JevAnnotator(model.config)
+        annotator._system_one = model._system_one
+        annotator.stats = model.stats
+        return annotator
+    raise TypeError("model must be a Jev/Kev configuration, preset, System One client or factor annotator")
+
+
+def annotate(texts, definitions: Mapping[str, str | MeasurementSpec], *, model,
+             concurrency: int = 16, batch_size: int = 8) -> AnnotationResult:
+    """Measure fixed definitions in row order, without judging or deduplicating them.
+
+    Each request contains one text and up to batch_size factors. A pandas Series
+    retains its index, including repeated labels. Transport failures propagate;
+    a mention gate can independently record a missing measurement as NaN.
+    """
+    if not isinstance(concurrency, int) or isinstance(concurrency, bool) or concurrency < 1:
+        raise ValueError("concurrency must be a positive integer")
+    if not isinstance(batch_size, int) or isinstance(batch_size, bool) or not 1 <= batch_size <= 8:
+        raise ValueError("batch_size must be between 1 and 8")
+    if isinstance(texts, str):
+        raise TypeError("texts must be a sequence, not a single string")
+    series = texts.copy() if isinstance(texts, pd.Series) else pd.Series(list(texts), dtype=object)
+    if series.empty or not series.map(lambda text: isinstance(text, str) and bool(text.strip())).all():
+        raise ValueError("texts must contain nonempty strings")
+    if not isinstance(definitions, Mapping) or not definitions:
+        raise ValueError("definitions must map factor names to definitions")
+    specs = {}
+    for name, definition in definitions.items():
+        spec = definition if isinstance(definition, MeasurementSpec) else factor_definition_to_spec(name, definition)
+        if spec.factor_name != name:
+            raise ValueError("Definition names must match their mapping keys")
+        specs[name] = spec.validate()
+    backend = resolve_annotator(model)
+    prepared = {name: backend.prepare_factor(name, spec) for name, spec in specs.items()}
+    names = list(prepared)
+
+    def measure(item):
+        row, text = item
+        values, metadata = {}, []
+        for start in range(0, len(names), batch_size):
+            chunk = {name: prepared[name] for name in names[start:start + batch_size]}
+            answers = backend.annotate_record(text, chunk, sample_id=row)
+            for name in chunk:
+                answer = answers[name]
+                values[name] = answer.value
+                snapshot = getattr(backend, "annotation_metadata", None)
+                record = snapshot(row, name) if callable(snapshot) else None
+                metadata.append({**(record if record is not None else answer.to_metadata()), "row": row})
+        return values, metadata
+
+    with ThreadPoolExecutor(max_workers=concurrency) as pool:
+        measured = list(pool.map(measure, enumerate(series.tolist())))
+    values = pd.DataFrame([row for row, _ in measured], index=series.index, columns=names)
+    metadata = pd.DataFrame([record for _, records in measured for record in records])
+    return AnnotationResult(values, metadata, dict(prepared))

coatlearn/annotation/dedup.py

@@ -0,0 +1,135 @@
+"""Jev concept-level comparison, usable standalone or via opt-in COAT dedup.
+
+Compare original definitions without rewriting them or reusing labels across
+non-equivalent rubrics. UNCERTAIN and technical ERROR both retain a candidate,
+but are distinct audit outcomes. This is not a definition/label merge engine.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict
+
+from coatlearn.annotation.types import MeasurementSpec
+from coatlearn.models.jev import JevConfig
+
+DEFAULT_TASK_CONTEXT = "从非结构化文本中识别候选因子；比较完整定义对应的核心属性，而非预测目标值或判断因果关系。"
+
+
+def normalize_factor_name(name: str) -> str:
+    """Legacy exact-name normalization only; no substring or WordNet veto."""
+    return " ".join(name.lower().replace("_", " ").replace("-", " ").split())
+
+
+def configured_deduplicator(fields):
+    """Validate optional core config without constructing an HTTP client."""
+    if fields is None:
+        return None, DEFAULT_TASK_CONTEXT
+    if not isinstance(fields, dict) or fields.get("backend") not in {"jev", "kev"}:
+        raise ValueError("factor_dedup requires backend='jev' or 'kev'")
+    fields = dict(fields)
+    context = fields.pop("task_context", DEFAULT_TASK_CONTEXT)
+    if not isinstance(context, str) or not context.strip():
+        raise ValueError("factor_dedup.task_context must be a nonempty string")
+    from coatlearn.models.kev import decision_config
+    return JevFactorDeduplicator(decision_config(fields)), context
+
+
+def core_dedup_policy(comparator, context):
+    """Record actual orchestration separately from the standalone pair policy."""
+    if comparator is None:
+        return None
+    return {**comparator.policy(), "integrated_into_coat": True,
+            "task_context": context, "comparison_order": "existing_A_new_B",
+            "simple_rules": ["exact_normalized_name", "cross_cluster_exact_definition"],
+            "representative": "first_valid_measured_factor",
+            "candidate_order": "cluster_index_then_proposal_order",
+            "reverse_check": False, "confidence_threshold": None}
+
+DEDUP_INSTRUCTIONS = """判断两个候选因子是否描述同一个核心属性。
+同一属性的措辞、评价角度、取值范围或判定细节有所变化，不足以构成新的独立因子。例如，属性本身与针对该属性的评价，可以属于概念重复。
+但仅仅讨论同一对象、主题相关、存在因果联系或统计相关，不构成重复。不同主体的评价或不同结果维度，也不能仅因相关而合并。
+比较两份完整定义，不仅比较名称。
+不改写定义，不判断哪个定义更好。"""
+
+
+def dedup_question() -> dict:
+    return {
+        "type": "choice",
+        "instructions": DEDUP_INSTRUCTIONS,
+        "criteria": {
+            "DUPLICATE": "能够明确判断为同一核心属性的重复表达。",
+            "DISTINCT": "能够明确指出不同的核心属性。",
+            "UNCERTAIN": "现有定义不足以明确判断。",
+        },
+    }
+
+
+def build_state(definition_a: str, definition_b: str, task_context: str) -> str:
+    """Only task context and verbatim definitions enter state; no reference labels."""
+    for text in (definition_a, definition_b, task_context):
+        if not isinstance(text, str) or not text.strip():
+            raise ValueError("Two nonempty definitions and task context are required")
+    return ("# Task context\n\n" + task_context
+            + "\n\n# Factor A (verbatim)\n\n" + definition_a
+            + "\n\n# Factor B (verbatim)\n\n" + definition_b)
+
+
+class JevFactorDeduplicator:
+    """One native Choice request per pair; no main LLM or discovery calls.
+
+    Original request and response are local audit artifacts, not public results.
+    No confidence cutoff, automatic retries beyond config, or label rewriting.
+    """
+
+    def __init__(self, config: JevConfig):
+        from coatlearn.annotation.measurement import decision_annotator
+        self.backend = decision_annotator(config)
+        question = dedup_question()
+        self.spec = MeasurementSpec(
+            factor_name="relation", jev_type="choice",
+            instructions=question["instructions"],
+            criteria=tuple(question["criteria"].items()),
+            choice_values={"DUPLICATE": 1, "DISTINCT": 0, "UNCERTAIN": -1},
+        ).validate()
+
+    def policy(self) -> dict:
+        backend = self.backend.config.redacted().get("backend", "jev")
+        return {"version": "concept-dedup-v1", "backend": backend,
+                "config": self.backend.config.redacted(),
+                "questions": {"relation": dedup_question()},
+                "uncertain_action": "retain", "error_action": "retain",
+                "rewrites_definitions": False, "integrated_into_coat": False}
+
+    def stats(self) -> dict:
+        return asdict(self.backend.stats())
+
+    def evaluate(self, definition_a: str, definition_b: str, task_context: str) -> dict:
+        request = {"model": self.backend.config.model,
+                   "state": build_state(definition_a, definition_b, task_context),
+                   "questions": {"relation": dedup_question()}}
+        result = {"status": "error", "decision": "ERROR", "action": "retain",
+                  "request": request, "response": None}
+        try:
+            response = self.backend._system_one(request)
+            result["response"] = response
+            if (response.get("model") != self.backend.config.model
+                    or set(response.get("answers", {})) != {"relation"}):
+                raise ValueError("Unexpected comparator model or answer keys")
+            answer = self.backend._interpret_answer(
+                "relation", self.spec, response["answers"]["relation"], None)
+            decision = answer.raw["choice"]
+            result.update(status="ok", decision=decision,
+                          action="deduplicate" if decision == "DUPLICATE" else "retain",
+                          probabilities=answer.probabilities, confidence=answer.confidence)
+        except Exception as exc:
+            # Transport exceptions may contain secrets: record type only.
+            result["error_type"] = type(exc).__name__
+        return result
+
+
+def compare_definitions(definition_a, definition_b, *, model, task_context=DEFAULT_TASK_CONTEXT):
+    """Compare two full definitions without rewriting them or running discovery."""
+    from .batch import resolve_annotator
+    backend = resolve_annotator(model)
+    comparator = JevFactorDeduplicator(backend.config)
+    comparator.backend = backend
+    return comparator.evaluate(definition_a, definition_b, task_context)

coatlearn/annotation/definitions.py

@@ -0,0 +1,216 @@
+"""Deterministic translation of COAT's extracted ternary factor definitions."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from typing import Any
+
+from coatlearn.annotation.types import MeasurementSpec, UnsupportedMeasurement
+
+__all__ = ["factor_definition_to_spec", "general_definition_to_spec"]
+
+_GENERAL_TYPE = re.compile(r"^[-+*•]?\s*type\s*:\s*(score|choice)\s*$", re.I)
+_VALUES = re.compile(r"^[-+*•]?\s*values\s*:\s*(.+)$", re.I)
+_MENTIONED = re.compile(r"^[-+*•]?\s*mentioned\s+when\s*:\s*(.+)$", re.I)
+_LEVEL = re.compile(r"^[-+*•]?\s*(\d+)\s*:\s*(.+)$")
+_OPTION = re.compile(r"^[-+*•]?\s*([^:]+?)\s*:\s*(.+)$")
+
+
+def _unwrap(text: str) -> str:
+    text = text.strip()
+    while True:
+        for marker in ("**", "__", "`", "*", "_"):
+            if (
+                text.startswith(marker)
+                and text.endswith(marker)
+                and len(text) >= 2 * len(marker)
+                and text.find(marker, len(marker)) == len(text) - len(marker)
+            ):
+                text = text[len(marker) : -len(marker)].strip()
+                break
+        else:
+            return text
+
+
+def _unbullet(text: str) -> str:
+    return re.sub(r"^(?:[-+*•]\s+|\d+[.)]\s+)", "", text.strip()).strip()
+
+
+def _name(text: str) -> str:
+    text = re.sub(r"^#{1,6}\s+", "", _unbullet(_unwrap(text)))
+    return " ".join(_unwrap(text.rstrip(":")).rstrip(":").split())
+
+
+def factor_definition_to_spec(factor_name: str, factor_definition: Any) -> MeasurementSpec:
+    """Parse exactly one named {-1, 0, 1} factor without inference or requests.
+
+    Accept the extracted four-line text or its exact ``name``/``description``
+    record. Markdown decoration and whitespace are presentation only; criterion
+    wording and label associations are retained. Unsupported input raises
+    :class:`UnsupportedMeasurement` before any backend is involved.
+    """
+    if not isinstance(factor_name, str) or not factor_name.strip():
+        raise UnsupportedMeasurement(factor_name, "factor_name must be a nonempty string")
+    name = _name(factor_name)
+    if not name:
+        raise UnsupportedMeasurement(factor_name, "factor_name must contain a name")
+    if isinstance(factor_definition, Mapping):
+        if set(factor_definition) != {"name", "description"}:
+            raise UnsupportedMeasurement(factor_name, "expected exactly name and description")
+        record_name = factor_definition["name"]
+        if not isinstance(record_name, str) or _name(record_name).casefold() != name.casefold():
+            raise UnsupportedMeasurement(factor_name, "factor record name does not match")
+        factor_definition = factor_definition["description"]
+    if not isinstance(factor_definition, str):
+        raise UnsupportedMeasurement(factor_name, "factor definition must be text or a factor record")
+    if any(_GENERAL_TYPE.match(line.strip()) for line in factor_definition.splitlines()):
+        return general_definition_to_spec(factor_name, factor_definition)
+    lines = [line.strip() for line in factor_definition.splitlines() if line.strip()]
+    if len(lines) >= 2 and lines[0] in ("```", "```text", "```markdown") and lines[-1] == "```":
+        lines = lines[1:-1]
+    if len(lines) != 4:
+        raise UnsupportedMeasurement(factor_name, "expected a name and exactly three criterion lines")
+    if _name(lines[0]).casefold() != name.casefold():
+        raise UnsupportedMeasurement(factor_name, "definition name does not match factor_name")
+    criteria = {}
+    for line in lines[1:]:
+        line = _unwrap(_unbullet(line))
+        line = re.sub(r"^(\*\*|__|`|\*|_)([-−]?\d+\s*:?)\1", r"\2", line)
+        match = re.fullmatch(r"([-−]?\d+)\s*:\s*(.*)", line)
+        if match is None:
+            raise UnsupportedMeasurement(factor_name, "expected a numeric label and criterion")
+        label = match.group(1).replace("−", "-")
+        description = " ".join(match.group(2).split())
+        if label not in ("1", "0", "-1") or label in criteria:
+            raise UnsupportedMeasurement(factor_name, "labels must be exactly 1, 0 and -1 without duplicates")
+        if not _unwrap(description):
+            raise UnsupportedMeasurement(factor_name, "criteria must not be empty")
+        criteria[label] = description
+    if set(criteria) != {"1", "0", "-1"}:
+        raise UnsupportedMeasurement(factor_name, "labels must be exactly 1, 0 and -1")
+    return MeasurementSpec(
+        factor_name=factor_name,
+        jev_type="choice",
+        instructions=f"Factor: {name}",
+        criteria=tuple((label, criteria[label]) for label in ("1", "0", "-1")),
+        choice_values={"1": 1.0, "0": 0.0, "-1": -1.0},
+        source="parsed",
+        notes="COAT ternary criteria parsed verbatim; value is the selected label, not probability or confidence.",
+    ).validate()
+
+
+def general_definition_to_spec(factor_name: str, factor_definition: str) -> MeasurementSpec:
+    """Parse a general-template factor with an explicit measurement type.
+
+    The proposal declares ``Type: score|choice``, ``Values: ...`` and
+    ``Mentioned when: ...``; the mention sentence becomes a noul gate so that
+    "text does not mention this attribute" is recorded as missing rather than
+    folded into a value. Parsing is deterministic and makes no model calls.
+    """
+    if not isinstance(factor_name, str) or not factor_name.strip():
+        raise UnsupportedMeasurement(factor_name, "factor_name must be a nonempty string")
+    name = _name(factor_name)
+    if not name:
+        raise UnsupportedMeasurement(factor_name, "factor_name must contain a name")
+    if not isinstance(factor_definition, str):
+        raise UnsupportedMeasurement(factor_name, "factor definition must be text")
+    lines = [line.strip() for line in factor_definition.splitlines() if line.strip()]
+    if len(lines) >= 2 and lines[0] in ("```", "```text", "```markdown") and lines[-1] == "```":
+        lines = lines[1:-1]
+    if len(lines) < 4 or _name(lines[0]).casefold() != name.casefold():
+        raise UnsupportedMeasurement(factor_name, "expected a name line and a general definition")
+    kind = values_text = mentioned = None
+    body = []
+    for line in lines[1:]:
+        stripped = _unwrap(_unbullet(line))
+        match = _GENERAL_TYPE.match(stripped)
+        if match and kind is None:
+            kind = match.group(1).lower()
+            continue
+        match = _VALUES.match(stripped)
+        if match and values_text is None:
+            values_text = match.group(1).strip()
+            continue
+        match = _MENTIONED.match(stripped)
+        if match and mentioned is None:
+            mentioned = " ".join(match.group(1).split())
+            continue
+        body.append(stripped)
+    if kind is None or values_text is None or mentioned is None:
+        raise UnsupportedMeasurement(
+            factor_name, "general definition needs Type, Values and Mentioned when"
+        )
+    if not _unwrap(mentioned):
+        raise UnsupportedMeasurement(factor_name, "Mentioned when must not be empty")
+
+    gate = MeasurementSpec(
+        factor_name=factor_name,
+        jev_type="noul",
+        instructions=f"Does the text explicitly mention this attribute? Factor: {name}",
+        criteria=(mentioned, "The text does not explicitly mention this attribute."),
+        source="parsed",
+        notes="Mention gate: text without an explicit statement is recorded as missing.",
+    )
+
+    if kind == "score":
+        try:
+            value_map = tuple(float(v.strip()) for v in values_text.split(","))
+        except ValueError:
+            raise UnsupportedMeasurement(factor_name, "score Values must be comma-separated numbers") from None
+        levels = {}
+        for line in body:
+            match = _LEVEL.match(line)
+            if match is None:
+                raise UnsupportedMeasurement(factor_name, "expected numbered score levels")
+            index = int(match.group(1))
+            if index in levels:
+                raise UnsupportedMeasurement(factor_name, "duplicate score level")
+            levels[index] = " ".join(match.group(2).split())
+        if len(value_map) < 2 or len(levels) != len(value_map):
+            raise UnsupportedMeasurement(
+                factor_name, "score needs one level per Values entry")
+        # Level labels may be 0..n-1 or the numeric values themselves; either way
+        # they are ordered ascending and paired with Values in that order.
+        ordered = sorted(levels)
+        return MeasurementSpec(
+            factor_name=factor_name,
+            jev_type="score",
+            instructions=f"Factor: {name}",
+            criteria=tuple(levels[key] for key in ordered),
+            value_map=value_map,
+            source="parsed",
+            notes="General score parsed verbatim; value is the probability-weighted expectation over Values.",
+            gate=gate,
+        ).validate()
+
+    options = {}
+    for line in body:
+        match = _OPTION.match(line)
+        if match is None:
+            raise UnsupportedMeasurement(factor_name, "expected named choice options")
+        option, description = match.group(1).strip(), " ".join(match.group(2).split())
+        if option in options:
+            raise UnsupportedMeasurement(factor_name, "duplicate choice option")
+        options[option] = description
+    choice_values = {}
+    for item in values_text.split(","):
+        if "=" not in item:
+            raise UnsupportedMeasurement(factor_name, "choice Values must use option=number")
+        option, value = item.split("=", 1)
+        try:
+            choice_values[option.strip()] = float(value.strip())
+        except ValueError:
+            raise UnsupportedMeasurement(factor_name, "choice value must be numeric") from None
+    if set(choice_values) != set(options):
+        raise UnsupportedMeasurement(factor_name, "Values must cover exactly the declared options")
+    return MeasurementSpec(
+        factor_name=factor_name,
+        jev_type="choice",
+        instructions=f"Factor: {name}",
+        criteria=tuple((option, options[option]) for option in options),
+        choice_values=choice_values,
+        source="parsed",
+        notes="General choice parsed verbatim; value is the selected option's declared number.",
+        gate=gate,
+    ).validate()

coatlearn/annotation/judge.py

@@ -0,0 +1,117 @@
+"""Opt-in Jev factor-definition judge; separate from per-review annotation.
+
+One native request, two independent Choice questions, both must PASS. The
+wording-clarity instruction is the author's exact text. No schema/main LLM,
+confidence threshold, or implicit rewrite of the proposed definition is used.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict
+from typing import Any
+
+from coatlearn.annotation.types import MeasurementSpec, UnsupportedMeasurement
+from coatlearn.models.jev import JevConfig
+from coatlearn.annotation.definitions import factor_definition_to_spec
+
+CLARITY_INSTRUCTIONS = "结合任务示例，因子名称和各取值说明是否有明确、可理解的含义？仅因实质性的文字歧义拒绝，例如指代不明、修饰语或否定范围存在不同合理解释。"
+FORMAT_INSTRUCTIONS = "因子定义的格式是否正确：一个非空的因子名称，随后是标记为 1、0、-1 的三条非空取值说明，三个标签各出现一次且没有额外取值或内容？Markdown 修饰和空白不影响格式。"
+FORMAT_INSTRUCTIONS_GENERAL = "因子定义的格式是否正确：一个非空的因子名称，随后声明 Type（score 或 choice）、Values、Mentioned when，并且每个取值或选项都有非空的说明？Markdown 修饰和空白不影响格式。"
+
+
+def judge_questions(template: str = "simple") -> dict:
+    """Fresh native question objects, safe to freeze in audit/provenance."""
+    format_instructions = (
+        FORMAT_INSTRUCTIONS_GENERAL if template == "general" else FORMAT_INSTRUCTIONS
+    )
+    return {
+        "clarity": {"type": "choice", "instructions": CLARITY_INSTRUCTIONS, "criteria": {
+            "PASS": "因子名称和各取值说明有明确、可理解的含义。",
+            "REJECT": "存在实质性的文字歧义。"}},
+        "format": {"type": "choice", "instructions": format_instructions, "criteria": {
+            "PASS": "格式正确。", "REJECT": "格式不正确。"}},
+    }
+
+
+def proposal_examples(prompt: str) -> str:
+    """Extract the actual data section, never regenerate/resample examples."""
+    examples = prompt.split("# Data\n\n", 1)[1].split("# Tasks: Factor Abstraction.", 1)[0]
+    for heading in ("\n# Existing Factors", "\n# Previously Rejected Factors"):
+        examples = examples.split(heading, 1)[0]
+    return examples
+
+
+def configured_judge(fields, template="simple"):
+    """Choose a typed backend without changing judge questions or acceptance."""
+    if fields is None:
+        return None
+    if not isinstance(fields, dict) or fields.get("backend") not in {"jev", "kev"}:
+        raise ValueError("factor_judge requires backend='jev' or 'kev'")
+    from coatlearn.models.kev import decision_config
+    return JevFactorJudge(decision_config(fields), template=template)
+
+
+class JevFactorJudge:
+    """Reuse the existing native transport and strict Choice response parser.
+
+    ``evaluate`` returns a local audit record, which can contain example text
+    and final responses. It must NOT be put into the public result dictionary.
+    API/response errors are ERROR, not a semantic rejection or a normal zero.
+    """
+
+    def __init__(self, config: JevConfig, template: str = "simple"):
+        from coatlearn.annotation.measurement import decision_annotator
+        self.backend = decision_annotator(config)
+        self.template = template
+        self.specs = {
+            name: MeasurementSpec(factor_name=name, jev_type="choice", instructions=q["instructions"],
+                                  criteria=tuple(q["criteria"].items()), choice_values={"PASS": 1, "REJECT": 0}).validate()
+            for name, q in judge_questions(template).items()
+        }
+
+    def policy(self) -> dict:
+        backend = self.backend.config.redacted().get("backend", "jev")
+        return {"backend": backend, "version": "clarity-format-v1", "template": self.template,
+                "config": self.backend.config.redacted(),
+                "questions": judge_questions(self.template), "acceptance": "both_PASS", "trials": 1,
+                "replaces": "legacy_quality_prescoring_and_three_LLM_judges"}
+
+    def stats(self) -> dict:
+        return asdict(self.backend.stats())
+
+    def evaluate(self, factor_name: str, definition: str, examples: str) -> dict[str, Any]:
+        result = {"status": "ok", "decision": "REJECT", "local_format": "PASS", "response": None,
+                  "request": {"model": self.backend.config.model,
+                              "state": "# Candidate factor definition (verbatim)\n\n" + definition
+                              + "\n\n# Original proposal examples (verbatim data section)\n" + examples,
+                              "questions": judge_questions(self.template)}}
+        try:
+            factor_definition_to_spec(factor_name, definition)
+        except UnsupportedMeasurement:
+            result.update(local_format="REJECT", rejected_checks=["local_format"])
+            return result
+        try:
+            if not isinstance(examples, str) or not examples.strip():
+                raise ValueError("Original proposal examples are required")
+            response = self.backend._system_one(result["request"])
+            result["response"] = response  # retain even malformed final JSON in local audit
+            if response.get("model") != self.backend.config.model or set(response.get("answers", {})) != set(self.specs):
+                raise ValueError("Unexpected judge model or answer keys")
+            answers = {}
+            for name, spec in self.specs.items():
+                annotation = self.backend._interpret_answer(name, spec, response["answers"][name], None)
+                answers[name] = {"decision": annotation.raw["choice"], "probabilities": annotation.probabilities,
+                                 "confidence": annotation.confidence}
+            rejected = [name for name, a in answers.items() if a["decision"] == "REJECT"]
+            result.update(answers=answers, rejected_checks=rejected, decision="REJECT" if rejected else "PASS")
+        except Exception as exc:
+            result.update(status="error", decision="ERROR", error_type=type(exc).__name__)
+        return result
+
+
+def review_definition(name, definition, *, examples, model, template="general"):
+    """Review one definition independently of annotation or discovery."""
+    from .batch import resolve_annotator
+    backend = resolve_annotator(model)
+    judge = JevFactorJudge(backend.config, template=template)
+    judge.backend = backend
+    return judge.evaluate(name, definition, examples)