premanmcp 1.1.11 → 1.1.13

package/bin/eval_generation_debug.py

@@ -0,0 +1,279 @@
+"""Opt-in evidence for ASSERT test-set generation; offline replay and bounded retry.
+
+Enable through run_assert.py with PREMAN_EVAL_DEBUG_DIR. Captures stay local;
+stdout/stderr contain summaries and paths, never request or response bodies.
+Keep preman-mcp/bin/eval_generation_debug.py byte-identical to this file.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from datetime import datetime, timezone
+from functools import wraps
+import importlib.metadata
+import json
+import logging
+import os
+from pathlib import Path
+import re
+import runpy
+import sys
+import uuid
+
+log = logging.getLogger("preman.eval_generation")
+# Share names across structured fields and malformed/embedded JSON strings.
+_SECRET_NAMES = (
+    r"(?:authorization|proxy[-_]authorization|[\w-]*api[-_]?key|"
+    r"[\w-]*password[\w-]*|[\w-]*secret[\w-]*|[\w-]*credential[\w-]*|"
+    r"[\w-]*token|cookie|set[-_]cookie)"
+)
+_SECRET_FIELD = re.compile(_SECRET_NAMES, re.I)
+_TEXT_SECRET = re.compile(
+    r"""(\b""" + _SECRET_NAMES + r"""["']?\s*[:=]\s*)(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;}]+)""", re.I
+)
+_HEADER_SECRET = re.compile(
+    r"(?i)(\b(?:authorization|proxy-authorization|cookie|set-cookie)\s*:\s*)[^\r\n]+"
+)
+
+
+def _diagnose(parsed, schema):
+    """Mirror the pinned stage's acceptance rules; reuse its case normalizer."""
+    from assert_ai.stages.test_set import normalize_generated_test_case
+
+    array = schema["properties"]["test_set"]
+    count = array.get("minItems", 1)
+    result = {"valid": False, "reason": "invalid_test_set_payload", "expected": count}
+    if not isinstance(parsed, dict) or not isinstance(parsed.get("test_set"), list):
+        return result
+    cases = parsed["test_set"]
+    result["received"] = len(cases)
+    if len(cases) < count:
+        return {**result, "reason": "too_few_cases"}
+    tool_source = "per_test_case" if "tools" in array["items"]["properties"] else "runtime"
+    for index, case in enumerate(cases[:count]):
+        try:
+            if not isinstance(case, dict):
+                raise ValueError("case is not an object")
+            normalize_generated_test_case(case, tool_source=tool_source, fixed_system_prompt=None)
+        except Exception as exc:
+            # A malformed tool can make the upstream normalizer raise AttributeError.
+            # That rejected reply is evidence too; never lose it during diagnosis.
+            return {**result, "reason": "invalid_case", "case_index": index, "error_type": type(exc).__name__}
+    return {**result, "valid": True, "reason": "accepted"}
+
+
+def _redact(value):
+    """Remove credential fields/patterns and literal credentials from the environment."""
+    from assert_ai.core.session import _sanitize_response_text
+
+    secrets = sorted({
+        value for key, value in os.environ.items()
+        if len(value) >= 6 and re.search(r"KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL", key, re.I)
+    }, key=len, reverse=True)
+    changed = False
+
+    def visit(item):
+        nonlocal changed
+        if isinstance(item, dict):
+            out = {}
+            for key, val in item.items():
+                if _SECRET_FIELD.fullmatch(key):
+                    out[key] = "[REDACTED]"
+                    changed = True
+                else:
+                    out[key] = visit(val)
+            return out
+        if isinstance(item, list):
+            return [visit(val) for val in item]
+        if isinstance(item, str):
+            clean = item
+            # Provider content often contains serialized JSON, so dict-only
+            # redaction cannot see its password/api_key fields.
+            try:
+                embedded = json.loads(clean)
+            except (ValueError, TypeError):
+                embedded = None
+            if isinstance(embedded, (dict, list)):
+                sanitized = visit(embedded)
+                if sanitized != embedded:
+                    clean = json.dumps(sanitized, ensure_ascii=False)
+            for secret in secrets:
+                clean = clean.replace(secret, "[REDACTED]")
+            clean = _TEXT_SECRET.sub(r'\1"[REDACTED]"', clean)
+            clean = _HEADER_SECRET.sub(r'\1[REDACTED]', clean)
+            clean = _sanitize_response_text(clean)
+            changed |= clean != item
+            return clean
+        return item
+
+    clean = visit(value)
+    return clean, changed
+
+
+def _capture(directory, model, messages, schema_name, schema, options, response):
+    from assert_ai.core.model_client import to_jsonable
+
+    result = _diagnose(response.parsed, schema)
+    dist = importlib.metadata.distribution("assert-ai")
+    source = json.loads(dist.read_text("direct_url.json") or "{}")
+    evidence = {
+        "format": "preman.eval-generation/v1",
+        "captured_at": datetime.now(timezone.utc).isoformat(),
+        "runtime": {"assert_ai_version": dist.version, "assert_ai_commit": source.get("vcs_info", {}).get("commit_id")},
+        "request": {"model": model, "messages": messages, "schema_name": schema_name, "json_schema": schema, "options": options},
+        "response": {name: getattr(response, name, None) for name in (
+            "text", "content", "parsed", "finish_reason", "status", "incomplete_details",
+            "model", "response_id", "usage", "api_mode", "raw",
+        )},
+        "result": result,
+    }
+    _, request_redacted = _redact(to_jsonable(evidence["request"]))
+    evidence, was_redacted = _redact(to_jsonable(evidence))
+    evidence["request_redacted"] = request_redacted
+    evidence["redacted"] = was_redacted
+    directory = Path(directory).expanduser().resolve()
+    directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+    status = "valid" if result["valid"] else "invalid"
+    path = directory / f"{status}-{uuid.uuid4().hex}.json"
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as stream:
+        json.dump(evidence, stream, ensure_ascii=False, indent=2)
+        stream.write("\n")
+    label = evidence["request"]["options"].get("call_label") if options is not None else None
+    (log.info if result["valid"] else log.warning)(
+        "generation status=%s batch=%s reason=%s capture=%s", status, label, result["reason"], path,
+    )
+    return path
+
+
+def install():
+    """Observe only test-set calls, without retrying or changing their responses."""
+    if not os.environ.get("PREMAN_EVAL_DEBUG_DIR"):
+        return
+    from assert_ai.stages import test_set
+
+    original = test_set.generate_structured
+    if getattr(original, "_preman_debug_capture", False):
+        return
+
+    @wraps(original)
+    async def observed(model, messages, *, schema_name, json_schema, options=None):
+        response = await original(model, messages, schema_name=schema_name, json_schema=json_schema, options=options)
+        directory = os.environ.get("PREMAN_EVAL_DEBUG_DIR")
+        if directory:
+            try:
+                _capture(directory, model, messages, schema_name, json_schema, options, response)
+            except Exception as exc:
+                # Evidence storage must never turn an accepted reply into a lost batch.
+                # Exception text can contain body data; log only its type.
+                log.warning("generation capture_failed error_type=%s", type(exc).__name__)
+        return response
+
+    observed._preman_debug_capture = True
+    test_set.generate_structured = observed
+
+
+def replay(capture):
+    """Reparse saved text and apply generation validation, with no model calls."""
+    from assert_ai.core.model_client import _maybe_parse_json
+
+    parsed = _maybe_parse_json(capture["response"]["text"] or "")
+    return {
+        **_diagnose(parsed, capture["request"]["json_schema"]),
+        "parser_matches_capture": parsed == capture["response"]["parsed"],
+        "redacted": capture["redacted"],
+        "finish_reason": capture["response"].get("finish_reason"),
+    }
+
+
+async def _retry(capture, attempts):
+    """Repeat a request sequentially; one provider attempt, no error-body logs."""
+    from assert_ai.core import model_client
+    from assert_ai.core.model_client import GenerateOptions
+    from assert_ai.stages import test_set
+
+    if capture.get("request_redacted", capture["redacted"]):
+        raise ValueError("Capture was redacted; refusing to send an altered request.")
+    request = capture["request"]
+    options = GenerateOptions(**(request["options"] or {}))
+    options.timeout_s = min(options.timeout_s or 120, 120)
+    directory = os.environ.get("PREMAN_EVAL_DEBUG_DIR", "artifacts/eval-generation-debug")
+    generate = test_set.generate_structured
+    # Other harness observers use wraps(), which copies this marker too.
+    # Remove the entire marked chain so this command captures exactly once.
+    while getattr(generate, "_preman_debug_capture", False):
+        generate = generate.__wrapped__
+
+    async def once(call_fn, **kwargs):
+        return await call_fn()
+
+    for attempt in range(1, attempts + 1):
+        original_retries = model_client._with_retries
+        disabled_level = logging.root.manager.disable
+        try:
+            # This standalone diagnostic command is sequential. Disable the
+            # upstream loop only during its request: it otherwise retries six
+            # times and logs provider exceptions, which can echo private bodies.
+            model_client._with_retries = once
+            logging.disable(logging.CRITICAL)
+            response = await asyncio.wait_for(generate(
+                request["model"], request["messages"], schema_name=request["schema_name"],
+                json_schema=request["json_schema"], options=options,
+            ), timeout=120)
+        except Exception as exc:
+            print(f"Live request failed ({type(exc).__name__}); no invalid payload reproduced.", file=sys.stderr)
+            return 2
+        finally:
+            model_client._with_retries = original_retries
+            logging.disable(disabled_level)
+        try:
+            path = _capture(directory, request["model"], request["messages"], request["schema_name"], request["json_schema"], options, response)
+        except Exception as exc:
+            print(f"Capture failed ({type(exc).__name__}); stopping the probe.", file=sys.stderr)
+            return 2
+        result = _diagnose(response.parsed, request["json_schema"])
+        print(json.dumps({"attempt": attempt, "capture": str(path), **result}), flush=True)
+        if not result["valid"]:
+            return 1
+    print("No invalid payload reproduced within the attempt limit.", flush=True)
+    return 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="command", required=True)
+    for command in ("inspect", "replay", "retry"):
+        child = sub.add_parser(command)
+        child.add_argument("capture", type=Path)
+        if command == "retry":
+            child.add_argument("--attempts", type=int, default=1)
+            child.add_argument("--output-dir", type=Path)
+    args = parser.parse_args(argv)
+    if args.command == "retry" and not 1 <= args.attempts <= 20:
+        parser.error("--attempts must be from 1 to 20")
+    try:
+        capture = json.loads(args.capture.read_text(encoding="utf-8"))
+        if capture.get("format") != "preman.eval-generation/v1":
+            raise ValueError("Unrecognized generation capture format")
+        if args.command == "inspect":
+            print(json.dumps(capture, indent=2, ensure_ascii=False))
+            return 0
+        if args.command == "replay":
+            result = replay(capture)
+            print(json.dumps(result, indent=2))
+            return 0 if result["valid"] else 1
+        directory = args.output_dir or args.capture.parent
+        os.environ["PREMAN_EVAL_DEBUG_DIR"] = str(directory.resolve())
+        # Reuse the same provider workarounds as a normal generation run.
+        parent = Path(__file__).resolve().parent
+        harness = parent / ("run_assert.py" if (parent / "run_assert.py").exists() else "eval_harness.py")
+        runpy.run_path(str(harness))
+        logging.basicConfig(level=logging.INFO)
+        return asyncio.run(_retry(capture, args.attempts))
+    except (OSError, ValueError, KeyError, TypeError) as exc:
+        parser.error(str(exc))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/bin/eval_generation_status.py

@@ -0,0 +1,284 @@
+"""Publish usable ASSERT cases and a separate, bounded generation error report.
+
+Keep the npm copy byte-identical. This adapter observes the pinned stage's actual
+job results, preserving its validation, retries, case IDs, and incomplete-cache
+signal. It does not synthesize missing cases or repeat any model requests.
+"""
+from __future__ import annotations
+
+from contextvars import ContextVar
+from functools import wraps
+import hashlib
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+import tempfile
+
+from eval_generation_contract import generation_spec_sha256
+
+from eval_generation_debug import _diagnose, _redact
+
+log = logging.getLogger('preman.eval_generation')
+_suite_root = ContextVar('preman_generation_suite', default=None)
+_spec_hash = ContextVar('preman_generation_spec_hash', default=None)
+_active = ContextVar('preman_generation', default=None)
+_kind = ContextVar('preman_generation_kind', default=None)
+_job = ContextVar('preman_generation_job', default=None)
+_MAX_FAILURES = 100
+
+
+def _failure(job, result):
+    """Use the stage's error verdict, with only safe response diagnostics."""
+    response = job.get('response')
+    reason = 'generation_error'
+    message = f"Generation failed ({type(result['error']).__name__})."
+    detail = {}
+    if response is not None:
+        diagnostic = _diagnose(response.parsed, job['schema'])
+        reason = diagnostic['reason'] if not diagnostic['valid'] else 'invalid_case'
+        message = {
+            'invalid_test_set_payload': 'The model did not return a valid test-case payload.',
+            'too_few_cases': 'The model returned fewer cases than this job requested.',
+            'invalid_case': 'The generated case did not pass validation.',
+        }.get(reason, message)
+        if response.finish_reason == 'length':
+            reason = 'output_token_limit'
+            message = 'The model reached its output token limit before returning valid cases.'
+        detail['finish_reason'] = response.finish_reason
+        # Redact before clipping so a cut credential cannot evade sanitization.
+        detail['response_text'] = _redact(response.text or '')[0][:8000]
+        usage = response.usage
+        if usage is not None:
+            detail['completion_tokens'] = usage.completion_tokens
+            raw = usage.raw if isinstance(usage.raw, dict) else {}
+            tokens = raw.get('completion_tokens_details') or raw.get('output_tokens_details') or {}
+            if isinstance(tokens, dict):
+                detail['reasoning_tokens'] = tokens.get('reasoning_tokens')
+    detail['max_tokens'] = job.get('max_tokens')
+    identity = _redact(job['identity'])[0]
+    identity['behavior'] = identity['behavior'][:256]
+    identity['dimensions'] = {str(k)[:128]: str(v)[:1024] for k, v in list(identity['dimensions'].items())[:32]}
+    return {**identity, 'reason': reason, 'message': message, **detail}
+
+
+def _atomic_write(path, content):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=f'.{path.name}-', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'wb') as stream:
+            stream.write(content)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def _publish(state, result):
+    from assert_ai.core.artifact_cache import _is_local_edit
+
+    version_bank = Path(result['test_set_path'])
+    suite = Path(_suite_root.get() or version_bank.parent)
+    bank = version_bank.read_bytes()
+    jobs = sorted(state['jobs'], key=lambda job: (job['identity']['kind'], job['order']))
+    failed = [job for job in jobs if job.get('error') is not None]
+    generated = int(result['saved_count'])
+    report = {
+        'format': 'preman.eval-generation-status/v1',
+        'status': 'partial' if failed and generated else 'failed' if failed else 'complete',
+        'expected_count': sum(job['identity']['case_count'] for job in jobs),
+        'generated_count': generated,
+        'failed_count': sum(job['identity']['case_count'] for job in failed),
+        'failure_job_count': len(failed),
+        'failures_truncated': len(failed) > _MAX_FAILURES,
+        'failures': [job['failure'] for job in failed[:_MAX_FAILURES]],
+        'bank_sha256': hashlib.sha256(bank).hexdigest(),
+        'generation_spec_sha256': _spec_hash.get(),
+    }
+    try:
+        report['taxonomy_sha256'] = hashlib.sha256(Path(state['taxonomy_path']).read_bytes()).hexdigest()
+    except OSError:
+        # Without a matching rubric the API declines reuse; ready cases and
+        # generation errors must still remain visible.
+        report['taxonomy_sha256'] = None
+    payload = (json.dumps(report, ensure_ascii=False, indent=2) + '\n').encode('utf-8')
+    _atomic_write(version_bank.parent / 'generation.json', payload)
+    if version_bank.resolve() != (suite / 'test_set.jsonl').resolve():
+        # Retain ASSERT's protection for edited/published banks. A partial
+        # bank has no complete-cache sidecar, so future generation cannot
+        # silently renumber its cases under the same historical identities.
+        if _is_local_edit(suite, 'test_set', suite / 'test_set.jsonl', version_bank):
+            log.warning('generation kept the existing bank and report; new output is at %s; use a new suite to publish different cases', version_bank)
+            return
+        _atomic_write(suite / 'test_set.jsonl', bank)
+    _atomic_write(suite / 'generation.json', payload)
+    log.info('generation status=%s generated=%d failed=%d report=%s', report['status'], generated, report['failed_count'], suite / 'generation.json')
+
+
+
+def _select_published_partial(ctx):
+    """Evaluate the bank offered to users, even beside an older complete cache."""
+    suite = Path(ctx['suite_root'])
+    bank = suite / 'test_set.jsonl'
+    try:
+        report = json.loads((suite / 'generation.json').read_text(encoding='utf-8'))
+        if not isinstance(report, dict) or report.get('status') != 'partial':
+            return
+        if report.get('bank_sha256') != hashlib.sha256(bank.read_bytes()).hexdigest():
+            return
+    except (OSError, ValueError):
+        return
+    ctx['test_set_path'] = str(bank)
+    # This is a usable partial bank, not the old cache version or a newly
+    # finalized artifact. Do not attribute its cases to that older version.
+    ctx.setdefault('artifact_versions', {}).pop('test_set', None)
+    ctx.pop('test_set_artifact_dir', None)
+    ctx.pop('stratification_path', None)
+    log.info('generation evaluating published partial bank %s', bank)
+
+
+def install():
+    """Adapt the pinned stage without modifying the installed ASSERT package."""
+    from assert_ai.stages import test_set
+    from assert_ai.core import artifact_cache
+    from assert_ai import config
+
+    original_context = config.load_runtime_context
+    if not getattr(original_context, '_preman_generation_status', False):
+        @wraps(original_context)
+        def load_context(raw, cfg_path, **kwargs):
+            # The runner applies CLI overrides before this call. Hash before
+            # ASSERT injects model defaults into raw's nested stage mappings.
+            fingerprint = generation_spec_sha256(raw, config_dir=Path(cfg_path).parent)
+            ctx = original_context(raw, cfg_path, **kwargs)
+            ctx['_preman_generation_spec_sha256'] = fingerprint
+            return ctx
+
+        load_context._preman_generation_status = True
+        config.load_runtime_context = load_context
+
+    original_activate = artifact_cache.activate_latest_artifacts
+    if not getattr(original_activate, '_preman_generation_status', False):
+        @wraps(original_activate)
+        def activate_latest(ctx):
+            original_activate(ctx)
+            _select_published_partial(ctx)
+
+        activate_latest._preman_generation_status = True
+        artifact_cache.activate_latest_artifacts = activate_latest
+    # runner imports this by name; support installation both before and after
+    # that import without importing runner (and loading dotenv) just to patch.
+    runner = sys.modules.get('assert_ai.runner')
+    if runner is not None:
+        runner.activate_latest_artifacts = artifact_cache.activate_latest_artifacts
+        runner.load_runtime_context = config.load_runtime_context
+
+    def patch(name, factory):
+        original = getattr(test_set, name)
+        if getattr(original, '_preman_generation_status', False):
+            return
+        wrapped = factory(original)
+        wrapped._preman_generation_status = True
+        setattr(test_set, name, wrapped)
+
+    def stage_wrapper(original):
+        @wraps(original)
+        async def run(ctx, raw_cfg):
+            token = _suite_root.set(ctx['suite_root'])
+            spec_token = _spec_hash.set(ctx.get('_preman_generation_spec_sha256'))
+            try:
+                return await original(ctx, raw_cfg)
+            finally:
+                _spec_hash.reset(spec_token)
+                _suite_root.reset(token)
+        return run
+
+    def orchestration_wrapper(original):
+        @wraps(original)
+        async def run_test_set(**kwargs):
+            state = {'jobs': [], 'taxonomy_path': kwargs['taxonomy_path']}
+            token = _active.set(state)
+            try:
+                result = await original(**kwargs)
+                _publish(state, result)
+                if not result['saved_count'] and result['errored_count']:
+                    raise ValueError('No test cases were generated; see generation.json for failed jobs')
+                return result
+            finally:
+                _active.reset(token)
+        return run_test_set
+
+    def records_wrapper(original):
+        @wraps(original)
+        async def generate_records(**kwargs):
+            token = _kind.set(kwargs['kind'])
+            try:
+                return await original(**kwargs)
+            except test_set.LLMAuthError:
+                raise
+            except Exception as exc:
+                state = _active.get()
+                jobs = [j for j in state['jobs'] if j['identity']['kind'] == kwargs['kind']] if state else []
+                # ASSERT raises the first batch error when a whole kind fails.
+                # Let the other kind finish, then decide failure for the bank.
+                # Planning/configuration errors do not satisfy this condition.
+                if jobs and all(j.get('error') is not None for j in jobs) and any(j['error'] is exc for j in jobs):
+                    return {'records': [], 'errored_count': len(jobs)}
+                raise
+            finally:
+                _kind.reset(token)
+        return generate_records
+
+    def gather_wrapper(original):
+        @wraps(original)
+        async def gather(items, *, limit, worker):
+            state, kind = _active.get(), _kind.get()
+            if state is None or kind is None:
+                return await original(items, limit=limit, worker=worker)
+
+            async def observed(item):
+                behavior = str(item.behavior.get('name') or '')
+                dimensions = {'behavior': behavior}
+                dimensions.update({name: str(level['name']) for name, level in (item.tuple_spec or {}).items()})
+                job = {
+                    'order': item.order,
+                    'identity': {
+                        'id': f'{kind}-generation-{item.order:06d}', 'kind': kind,
+                        'behavior': behavior, 'dimensions': dimensions, 'case_count': item.count,
+                    },
+                }
+                state['jobs'].append(job)
+                token = _job.set(job)
+                try:
+                    result = await worker(item)
+                    if 'error' in result:
+                        job['error'] = result['error']
+                        job['failure'] = _failure(job, result)
+                    return result
+                finally:
+                    # Keep bounded diagnostics only, not entire provider bodies.
+                    job.pop('response', None)
+                    job.pop('schema', None)
+                    _job.reset(token)
+            return await original(items, limit=limit, worker=observed)
+        return gather
+
+    def response_wrapper(original):
+        @wraps(original)
+        async def generate_structured(*args, **kwargs):
+            job = _job.get()
+            if job is not None:
+                job['schema'] = kwargs['json_schema']
+                job['max_tokens'] = getattr(kwargs.get('options'), 'max_tokens', None)
+            response = await original(*args, **kwargs)
+            if job is not None:
+                job['response'] = response
+            return response
+        return generate_structured
+
+    patch('run', stage_wrapper)
+    patch('run_test_set', orchestration_wrapper)
+    patch('_generate_records', records_wrapper)
+    patch('gather_limited', gather_wrapper)
+    patch('generate_structured', response_wrapper)