arango-query-core 0.2.2__tar.gz → 0.2.4__tar.gz

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/CHANGELOG.md

@@ -6,6 +6,52 @@ exact versions (contextual-data-fabric CC-9) and re-run their golden /
 eval suites on every bump — pre-1.0, **minor** bumps may break the API
 and this file must say so explicitly.
 
+## [0.2.4] - 2026-10-21
+
+### Added
+
+- `QueryLanguageAdapter` may define `normalize(query) -> str`, an optional seam
+  `NLQueryEngine` calls on every candidate after the fence is stripped and
+  before `validate`. Use it for a deterministic repair that would otherwise
+  cost an LLM retry; nl2cypher uses it to rewrite labels the schema does not
+  have. What it returns is validated, quoted in retries and returned. A
+  `normalize` that raises is logged and the candidate kept. It is not a
+  protocol member, so existing adapters are unaffected. #8
+- `NLQueryEngine.generate(..., user_hint="")` appends a caller hint to the
+  first attempt's user message only, for example "your previous query
+  failed with …" when re-asking. It leaves the system prompt (the cacheable
+  prefix), few-shot retrieval and postcondition context untouched. #10
+- `NLQueryEngine(retry_provider_errors=True)`: a provider call that raises (a
+  timeout, a 5xx) uses one attempt of `max_retries`, and the loop goes on.
+  `NLResult.provider_errors` counts them. If no attempt returned a response,
+  the last exception is re-raised. The default (`False`) keeps today's
+  behaviour: the first exception propagates. #9
+
+### Fixed
+
+- The engine finds the query in the first fenced block anywhere in the
+  response. It used to strip a fence only when the response started with
+  one, so "Here is the query: ```…```" failed validation and spent a retry.
+  A response that opens a fence without closing it yields what follows the
+  fence, and a response with no fence is returned as is, as before. When a
+  response has two fenced blocks, the first is taken; it used to be
+  everything from the first opening fence to the last closing one. #7
+
+## [0.2.3] - 2026-10-21
+
+### Added
+
+- `PostconditionViolation.soft` (default `False`): a soft violation is
+  retried like any other, but when the retry budget is spent
+  `NLQueryEngine.generate` returns the first valid candidate whose only
+  objection was soft, instead of failing closed. The candidate still passes
+  guardrails, and the violation stays on `NLResult.postcondition` (`ok` True
+  with a non-None `postcondition` marks a soft fallback). For checks such as
+  "the query returned no rows", where empty is sometimes the right answer.
+  `run_postconditions` now returns the first *hard* violation ahead of any
+  soft one, so a hard check listed after a soft one still applies. Additive;
+  existing checks are unaffected. Part of #4.
+
 ## [0.2.2] - 2026-10-21
 
 ### Fixed

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: arango-query-core
-Version: 0.2.2
+Version: 0.2.4
 Summary: Shared conceptual-model and NL substrate for ArangoDB query transpilers (arango-cypher-py, arango-sparql-py)
 Project-URL: Repository, https://github.com/ArthurKeen/arango-query-core
 Project-URL: Changelog, https://github.com/ArthurKeen/arango-query-core/blob/main/CHANGELOG.md

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/arango_query_core/nl/engine.py

@@ -17,6 +17,8 @@ signature breaks.
 
 from __future__ import annotations
 
+import logging
+import re
 from collections.abc import Sequence
 from dataclasses import dataclass, field
 from typing import Any
@@ -33,13 +35,18 @@ from .seams import GuardrailVerdict, QueryLanguageAdapter, ValidationResult
 
 _USAGE_KEYS = ("prompt_tokens", "completion_tokens", "total_tokens", "cached_tokens")
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class NLResult:
     """Outcome of one NL→query generation.
 
     ``query`` is empty when generation failed (``ok`` False); ``error``
-    then carries the last validation failure or guardrail refusal.
+    then carries the last validation failure, postcondition violation or
+    guardrail refusal. ``ok`` True with a non-None ``postcondition`` means a
+    soft fallback: the retry budget ran out and ``query`` is the first valid
+    candidate whose only objection was soft (``PostconditionViolation.soft``).
     Token fields accumulate across retries so cost accounting sees the
     whole conversation, not the last attempt.
     """
@@ -55,6 +62,8 @@ class NLResult:
     guardrail: GuardrailVerdict | None = None
     validation: ValidationResult | None = None
     postcondition: PostconditionViolation | None = None
+    provider_errors: int = 0
+    """Provider calls that raised and were retried (``retry_provider_errors``)."""
 
 
 @dataclass
@@ -88,17 +97,35 @@ class NLQueryEngine:
     guardrail_context: dict[str, Any] = field(default_factory=dict)
     postconditions: Sequence[Postcondition] = ()
     postcondition_mapping: Any = None
+    retry_provider_errors: bool = False
+    """When True, a provider call that raises (a timeout, a 5xx) uses one
+    attempt of the ``max_retries`` budget and the loop goes on; the count is
+    on :attr:`NLResult.provider_errors`. If no attempt ever returned a
+    response, the last exception is re-raised. False (the default) lets the
+    first exception propagate, as before."""
+
+    def generate(self, question: str, *, schema_context: str = "", user_hint: str = "") -> NLResult:
+        """Generate a query for *question*.
 
-    def generate(self, question: str, *, schema_context: str = "") -> NLResult:
+        *user_hint* is appended to the **first** attempt's user message only —
+        e.g. "your previous query failed with ..." when a caller re-asks after a
+        failure. It leaves the system prompt (the cacheable prefix), few-shot
+        retrieval and postcondition context untouched; retries carry their own
+        "previous query was rejected" text.
+        """
         result = NLResult()
         system = self._system_prompt(question, schema_context)
-        user = question
+        user = f"{question}\n\n{user_hint}" if user_hint else question
+        last_provider_error: Exception | None = None
         failure: ValidationResult | None = None
         violation: PostconditionViolation | None = None
         candidate = ""
+        # First valid candidate whose only objection was soft: what the call
+        # returns if the budget runs out without a clean candidate.
+        fallback: tuple[str, ValidationResult, PostconditionViolation] | None = None
 
         for attempt in range(self.max_retries + 1):
-            if attempt > 0:
+            if attempt > 0 and failure is not None:
                 # A postcondition rejection carries its own correction text;
                 # otherwise fall back to the adapter's syntactic repair hint.
                 if violation is not None:
@@ -113,11 +140,26 @@ class NLQueryEngine:
                     f"Problem: {hint}\n"
                     f"Return a corrected query only."
                 )
+            if attempt > 0:
                 result.retries = attempt
-            content, usage = self.provider.generate(system, user)
+            try:
+                content, usage = self.provider.generate(system, user)
+            except Exception as exc:
+                if not self.retry_provider_errors:
+                    raise
+                result.provider_errors += 1
+                last_provider_error = exc
+                logger.warning(
+                    "NL provider call failed (attempt %d/%d): %s: %s",
+                    attempt + 1,
+                    self.max_retries + 1,
+                    type(exc).__name__,
+                    exc,
+                )
+                continue
             for key in _USAGE_KEYS:
                 setattr(result, key, getattr(result, key) + int(usage.get(key, 0) or 0))
-            candidate = _strip_code_fence(content, self.adapter.language)
+            candidate = self._normalize(_strip_code_fence(content, self.adapter.language))
             failure = self.adapter.validate(candidate)
             result.validation = failure
             if not failure.ok:
@@ -138,13 +180,28 @@ class NLQueryEngine:
             result.postcondition = violation
             if violation is None:
                 break
+            # Soft only if no check objected as hard (run_postconditions
+            # returns a hard violation ahead of any soft one).
+            if violation.soft and fallback is None:
+                fallback = (candidate, failure, violation)
+        else:
+            # Budget spent without a clean candidate. If one drew only a soft
+            # objection, return it — with the objection on the result — rather
+            # than refuse a query that may well be right.
+            if fallback is not None:
+                candidate, result.validation, result.postcondition = fallback
+                failure, violation = result.validation, None
 
+        if failure is None and last_provider_error is not None:
+            # Every attempt raised: nothing to report but the failure itself.
+            raise last_provider_error
         if failure is None or not failure.ok:
             result.error = failure.error if failure else "provider returned nothing"
             return result
         if violation is not None:
-            # Budget spent on a valid-but-semantically-rejected query — fail
-            # closed rather than return a plausible wrong answer.
+            # Budget spent on a valid-but-semantically-rejected query with no
+            # soft-only candidate to fall back on — fail closed rather than
+            # return a plausible wrong answer.
             result.error = f"{violation.reason} ({violation.code})"
             return result
 
@@ -158,6 +215,21 @@ class NLQueryEngine:
         result.ok = True
         return result
 
+    def _normalize(self, candidate: str) -> str:
+        """The adapter's optional deterministic repair (``normalize`` seam),
+        applied before validation; the result is what is validated, quoted in
+        retries and returned. A ``normalize`` that raises is logged and the
+        candidate is kept — a broken repair must not abort generation."""
+        normalize = getattr(self.adapter, "normalize", None)
+        if not callable(normalize):
+            return candidate
+        try:
+            normalized = normalize(candidate)
+        except Exception as exc:  # noqa: BLE001 - see docstring
+            logger.warning("adapter.normalize raised %s: %s; keeping the candidate", type(exc).__name__, exc)
+            return candidate
+        return normalized if isinstance(normalized, str) else candidate
+
     def _system_prompt(self, question: str, schema_context: str) -> str:
         sections = [self.adapter.grammar_prompt_section(schema_context)]
         index = self.adapter.few_shot_index()
@@ -190,14 +262,22 @@ class NLQueryEngine:
         return "\n\n".join(s for s in sections if s)
 
 
+_FENCED_BLOCK = re.compile(r"```[^\n`]*\n(.*?)```", re.DOTALL)
+
+
 def _strip_code_fence(content: str, language: str) -> str:
     """Extract the query from a fenced code block, tolerating bare text.
 
     LLMs frequently wrap the answer in ```<lang> fences despite the
-    output-format contract; accept fenced, language-tagged-fenced, and
-    bare responses uniformly so the validator sees only query text.
+    output-format contract, and often lead with prose ("Here is the
+    query:") or follow with an explanation. The first fenced block wherever
+    it appears is the query; a response that only opens a fence yields what
+    follows it; a response without a fence is returned as is.
     """
     text = content.strip()
+    block = _FENCED_BLOCK.search(text)
+    if block is not None:
+        return block.group(1).strip()
     if not text.startswith("```"):
         return text
     first_newline = text.find("\n")

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/arango_query_core/nl/postconditions.py

@@ -15,14 +15,19 @@ budget as parse and translate: the model is told the rule up front (via
 :meth:`Postcondition.prompt_section`, rendered once into the cacheable system
 prefix), and a violation is fed back into the next prompt exactly like a
 validation failure — so the loop gets a chance to *correct* the query rather
-than just reject it, and fails closed when the budget is spent.
+than just reject it, and fails closed when the budget is spent — unless a
+candidate drew only a *soft* violation (``PostconditionViolation.soft``), in
+which case the first such candidate is returned instead.
 
 Semantics (mirrors ``nl2cypher``'s original, language-agnostic here):
 
 * Postconditions run **after** the adapter's ``validate`` succeeds — a retry is
   only ever spent on an already-valid query.
-* The **first** violation wins; checks are not accumulated. One clear, specific
-  correction beats a pile of them.
+* The **first hard** violation wins; checks are not accumulated. One clear,
+  specific correction beats a pile of them. A *soft* violation does not stop
+  the run: the remaining checks still see the query, and any hard violation
+  among them takes precedence. Only a query no check objects to as hard can
+  become a soft fallback, whatever order the checks are listed in.
 * A check that **raises** is treated as a violation, not an error — a broken
   postcondition must not turn a safety mechanism into an outage. Fail closed.
 * :meth:`Postcondition.prompt_section` renders **once**; it must not vary
@@ -60,6 +65,19 @@ class PostconditionViolation:
     """How to fix it. Also fed into the retry prompt, after ``reason``."""
     code: str = "postcondition_violation"
     """Stable identifier for telemetry / assertions, e.g. ``"time_windowed"``."""
+    soft: bool = False
+    """A soft violation asks for a retry but does not make the query unusable.
+
+    Use it for an objection that is sometimes the right answer — e.g. "the query
+    returned no rows" when the honest answer may be "none". When the retry
+    budget is spent, the engine returns the first valid candidate whose only
+    objection was soft (with this violation on ``NLResult.postcondition``)
+    instead of failing closed. Hard violations (the default) still fail closed.
+
+    Only that first soft candidate is offered to the guardrails: if they refuse
+    it, the call fails closed with the guardrail's reason, even when a later
+    soft candidate would have passed — the conservative reading.
+    """
 
 
 @dataclass(frozen=True)
@@ -109,15 +127,22 @@ def run_postconditions(
     postconditions: Sequence[Postcondition] | None,
     context: PostconditionContext,
 ) -> PostconditionViolation | None:
-    """Run *postconditions* in order, returning the first violation.
+    """Run *postconditions* in order, returning the first hard violation, or
+    else the first soft one.
+
+    A soft violation does not stop the run, so a hard check listed after it
+    still sees the query; otherwise a query breaking a hard rule could be
+    returned as a soft fallback.
 
-    A check that raises is treated as a violation rather than being allowed to
-    abort the translation: a broken check must not be able to turn a safety
-    mechanism into an outage, and failing closed is the conservative reading.
+    A check that raises is treated as a (hard) violation rather than being
+    allowed to abort the translation: a broken check must not be able to turn a
+    safety mechanism into an outage, and failing closed is the conservative
+    reading.
     """
     if not postconditions:
         return None
 
+    first_soft: PostconditionViolation | None = None
     for pc in postconditions:
         code = getattr(pc, "code", pc.__class__.__name__)
         try:
@@ -128,9 +153,13 @@ def run_postconditions(
                 reason=f"Postcondition {code!r} raised {type(exc).__name__}: {exc}",
                 suggested_hint="Simplify the query; the check could not evaluate it.",
             )
-        if violation is not None:
+        if violation is None:
+            continue
+        if not violation.soft:
             return violation
-    return None
+        if first_soft is None:
+            first_soft = violation
+    return first_soft
 
 
 def postcondition_prompt_sections(

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/arango_query_core/nl/seams.py

@@ -74,6 +74,15 @@ class QueryLanguageAdapter(Protocol):
     ``language`` is the lowercase tag used for fenced code blocks and
     telemetry (``"cypher"``, ``"sparql"``). All other members map
     one-to-one onto the numbered seams in the module docstring.
+
+    **Optional:** ``normalize(query: str) -> str``. When an adapter defines
+    it, :class:`~arango_query_core.nl.NLQueryEngine` calls it on every
+    candidate, after the code fence is stripped and before :meth:`validate`,
+    for a deterministic repair that would otherwise cost an LLM retry (e.g.
+    rewriting a label the schema does not have to the one it does). What it
+    returns is what is validated, quoted in a retry and returned. It is not a
+    protocol member, so existing adapters conform unchanged; a ``normalize``
+    that raises is logged and the candidate kept.
     """
 
     language: str

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/docs/PRD.md

@@ -122,14 +122,14 @@ to prevent a three-way fork.
 | `exec` | `AqlExecutor` protocol, `safe_execute` (bind-var reference checking), `explain_aql` |
 | `extensions` | `ExtensionRegistry` / `ExtensionPolicy` for `arango.*` extension functions |
 | `errors` | `CoreError` shared base |
-| `nl.engine` | `NLQueryEngine` — prompt assembly, few-shot retrieval, provider call, generate→validate→repair loop; `NLResult` |
+| `nl.engine` | `NLQueryEngine` — prompt assembly, few-shot retrieval, provider call, generate→validate→repair loop; `NLResult`. The query is the first fenced block anywhere in a response (prose around it is ignored), else the bare response. An adapter's optional `normalize` seam repairs each candidate before validation. `generate(user_hint=...)` appends a caller hint to the first user message only, leaving the cacheable system prefix alone. With `retry_provider_errors=True`, a provider call that raises uses one attempt of the budget (`NLResult.provider_errors`), and the last error is re-raised only if no attempt returned a response; the default propagates the first error. |
 | `nl.providers` | `LLMProvider` protocol; OpenAI / OpenRouter / Anthropic implementations (Anthropic with prompt-cache system-block splitting); env-driven `get_llm_provider` |
 | `nl.fewshot` | `FewShotIndex` over `(question, query)` pairs (legacy corpus keys accepted); `BM25Retriever` (rank_bm25, `nl` extra) and `DenseRetriever` (sentence-transformers, `dense` extra), selected via `mode="bm25"\|"dense"\|"auto"`; `cached_few_shot_index` memoizes the expensive dense build |
 | `nl.grounding` | Seam 6/7 retrieval: `LabelIndex`/`GroundedEntity` (entity grounding) and `PredicateIndex`/`GroundedPredicate` (predicate/schema-convention grounding, renders IRI local names not labels), sharing one token-substring scorer |
 | `nl.pathindex` | Seam 8: `ClassPathIndex`/`ClassPath` — bounded shortest predicate paths between two seam-6/7-anchored classes over the TBox's class-connectivity graph |
-| `nl.postconditions` | Caller-supplied semantic invariants (`Postcondition`, `PostconditionContext`, `PostconditionViolation`, `run_postconditions`) enforced through the engine's own retry budget; fails closed on exhaustion |
+| `nl.postconditions` | Caller-supplied semantic invariants (`Postcondition`, `PostconditionContext`, `PostconditionViolation`, `run_postconditions`) enforced through the engine's own retry budget; fails closed on exhaustion, unless a valid candidate drew only a soft violation (`PostconditionViolation.soft`); the first such candidate is returned, with the violation on `NLResult.postcondition` |
 | `nl.synthbank` | `SHAPE_CATALOG`/`ShapeTemplate` — 9 ontology-agnostic compositional query-shape templates for synthetic few-shot bank generation |
-| `nl.seams` | `QueryLanguageAdapter` — the eight language-specific seams: (1) grammar prompt section, (2) few-shot corpus, (3) syntax validator, (4) repair rules keyed on validator errors, (5) guardrail checks, (6) entity grounding index, (7) predicate grounding index, (8) relationship-path grounding index. Plus `ValidationResult`, `GuardrailVerdict` |
+| `nl.seams` | `QueryLanguageAdapter` — the eight language-specific seams: (1) grammar prompt section, (2) few-shot corpus, (3) syntax validator, (4) repair rules keyed on validator errors, (5) guardrail checks, (6) entity grounding index, (7) predicate grounding index, (8) relationship-path grounding index. Optional, outside the protocol: `normalize(query) -> str`, a deterministic repair applied before validation. Plus `ValidationResult`, `GuardrailVerdict` |
 
 ## 5. Requirements
 

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arango-query-core"
-version = "0.2.2"
+version = "0.2.4"
 description = "Shared conceptual-model and NL substrate for ArangoDB query transpilers (arango-cypher-py, arango-sparql-py)"
 readme = "README.md"
 license = { file = "LICENSE" }

arango_query_core-0.2.4/tests/test_nl_engine_extraction_and_resilience.py

@@ -0,0 +1,165 @@
+"""Engine behaviour arango-cypher-py needed to drop its own NL loop
+(arango-solutions-org/arango-cypher#24), one issue each:
+
+* #7 — the query is found in a fenced block anywhere in the response;
+* #8 — an adapter's optional ``normalize`` seam repairs a candidate before
+  validation;
+* #9 — ``retry_provider_errors`` lets a provider call that raised use the
+  retry budget instead of aborting generation;
+* #10 — ``generate(user_hint=...)`` adds a hint to the first user message.
+
+Fakes follow ``tests/test_nl_engine.py``: a scripted provider and an adapter
+that accepts any query containing ``SELECT``.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+from test_nl_engine import FakeAdapter  # tests/ is on sys.path (pytest rootdir import)
+
+from arango_query_core.nl import NLQueryEngine
+from arango_query_core.nl.engine import _strip_code_fence
+
+GOOD = "SELECT ?s WHERE { ?s ?p ?o }"
+
+
+class ScriptedProvider:
+    """Answers in order; an ``Exception`` in the script is raised instead."""
+
+    def __init__(self, *answers: str | Exception) -> None:
+        self._answers = list(answers)
+        self.calls: list[tuple[str, str]] = []
+
+    def generate(self, system: str, user: str) -> tuple[str, dict[str, int]]:
+        self.calls.append((system, user))
+        answer = self._answers.pop(0)
+        if isinstance(answer, Exception):
+            raise answer
+        return answer, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15, "cached_tokens": 0}
+
+
+# --- #7: the fenced query, wherever it is ---------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("response", "query"),
+    [
+        (f"Here is the query:\n```sparql\n{GOOD}\n```\nIt lists every triple.", GOOD),
+        (f"```\n{GOOD}\n```", GOOD),
+        (f"First:\n```sparql\n{GOOD}\n```\nOr:\n```sparql\nASK {{}}\n```", GOOD),
+        (f"```sparql\n{GOOD}", GOOD),  # a fence that is never closed
+        (GOOD, GOOD),  # no fence at all
+    ],
+    ids=["prose-around", "untagged", "first-of-two", "unclosed", "bare"],
+)
+def test_the_query_is_found_wherever_the_fence_is(response: str, query: str) -> None:
+    assert _strip_code_fence(response, "sparql") == query
+
+
+def test_prose_before_the_fence_costs_no_retry() -> None:
+    provider = ScriptedProvider(f"Sure! Here you go:\n```sparql\n{GOOD}\n```")
+    result = NLQueryEngine(provider=provider, adapter=FakeAdapter()).generate("everything")
+    assert result.ok and result.query == GOOD and result.retries == 0
+    assert len(provider.calls) == 1
+
+
+# --- #8: the normalize seam ------------------------------------------------------
+
+
+class NormalizingAdapter(FakeAdapter):
+    """Repairs a lowercase keyword the validator would reject."""
+
+    def __init__(self, *, fail: bool = False) -> None:
+        super().__init__()
+        self.seen: list[str] = []
+        self._fail = fail
+
+    def normalize(self, query: str) -> str:
+        self.seen.append(query)
+        if self._fail:
+            raise ValueError("normalizer bug")
+        return query.replace("select", "SELECT")
+
+
+def test_a_normalized_candidate_is_what_is_validated_and_returned() -> None:
+    adapter = NormalizingAdapter()
+    provider = ScriptedProvider("```sparql\nselect ?s WHERE { ?s ?p ?o }\n```")
+    result = NLQueryEngine(provider=provider, adapter=adapter).generate("everything")
+    assert result.ok and result.query == GOOD and result.retries == 0
+    assert adapter.seen == ["select ?s WHERE { ?s ?p ?o }"]  # after the fence is stripped
+
+
+def test_a_retry_quotes_the_normalized_candidate() -> None:
+    adapter = NormalizingAdapter()
+    provider = ScriptedProvider("ask {}", GOOD)
+    NLQueryEngine(provider=provider, adapter=adapter, max_retries=1).generate("everything")
+    assert "```sparql\nask {}\n```" in provider.calls[1][1]
+
+
+def test_a_normalize_that_raises_keeps_the_candidate(caplog: pytest.LogCaptureFixture) -> None:
+    provider = ScriptedProvider(GOOD)
+    with caplog.at_level(logging.WARNING):
+        result = NLQueryEngine(provider=provider, adapter=NormalizingAdapter(fail=True)).generate(
+            "everything"
+        )
+    assert result.ok and result.query == GOOD
+    assert "normalizer bug" in caplog.text
+
+
+# --- #9: provider failures and the retry budget ---------------------------------
+
+
+def test_by_default_a_provider_failure_propagates() -> None:
+    provider = ScriptedProvider(TimeoutError("timed out"), GOOD)
+    with pytest.raises(TimeoutError):
+        NLQueryEngine(provider=provider, adapter=FakeAdapter()).generate("everything")
+    assert len(provider.calls) == 1
+
+
+def test_opted_in_a_failed_call_uses_one_attempt() -> None:
+    provider = ScriptedProvider(TimeoutError("timed out"), GOOD)
+    engine = NLQueryEngine(provider=provider, adapter=FakeAdapter(), retry_provider_errors=True)
+    result = engine.generate("everything")
+    assert result.ok and result.query == GOOD
+    assert (result.provider_errors, result.retries, result.total_tokens) == (1, 1, 15)
+    assert provider.calls[1][1] == "everything"  # nothing to repair yet: the plain question again
+
+
+def test_opted_in_every_call_failing_reraises_the_last_error() -> None:
+    provider = ScriptedProvider(TimeoutError("first"), ConnectionError("second"), RuntimeError("third"))
+    engine = NLQueryEngine(
+        provider=provider, adapter=FakeAdapter(), max_retries=2, retry_provider_errors=True
+    )
+    with pytest.raises(RuntimeError, match="third"):
+        engine.generate("everything")
+    assert len(provider.calls) == 3
+
+
+def test_opted_in_a_failure_after_a_bad_candidate_reports_that_candidate() -> None:
+    provider = ScriptedProvider("ask {}", TimeoutError("timed out"))
+    engine = NLQueryEngine(
+        provider=provider, adapter=FakeAdapter(), max_retries=1, retry_provider_errors=True
+    )
+    result = engine.generate("everything")
+    assert not result.ok and result.error == "not a SELECT" and result.provider_errors == 1
+
+
+# --- #10: a hint on the first user message ---------------------------------------
+
+
+def test_a_user_hint_rides_on_the_first_message_only() -> None:
+    provider = ScriptedProvider("ask {}", GOOD)
+    engine = NLQueryEngine(provider=provider, adapter=FakeAdapter(), max_retries=1)
+    result = engine.generate("everything", user_hint="Your previous query timed out. Add LIMIT.")
+    assert result.ok
+    assert provider.calls[0][1] == "everything\n\nYour previous query timed out. Add LIMIT."
+    assert "timed out" not in provider.calls[1][1]
+
+
+def test_a_user_hint_leaves_the_system_prompt_alone() -> None:
+    plain, hinted = ScriptedProvider(GOOD), ScriptedProvider(GOOD)
+    NLQueryEngine(provider=plain, adapter=FakeAdapter()).generate("everything")
+    NLQueryEngine(provider=hinted, adapter=FakeAdapter()).generate("everything", user_hint="hint")
+    assert plain.calls[0][0] == hinted.calls[0][0]

{arango_query_core-0.2.2 → arango_query_core-0.2.4}/tests/test_nl_postconditions.py

@@ -4,7 +4,9 @@ Covers the semantics the feature promises: postconditions run only on valid
 queries, share the retry budget, feed their correction back into the next
 prompt, fail closed on exhaustion, first-violation-wins, a raising check is a
 violation (not an outage), the rule is announced once in the system prompt,
-and the guardrail gate still runs after. All with fakes — no network.
+and the guardrail gate still runs after. Soft violations: retried the same way,
+returned as a fallback on exhaustion, never ahead of a hard violation whatever
+the check order. All with fakes — no network.
 """
 
 from __future__ import annotations
@@ -208,3 +210,133 @@ def test_run_postconditions_helper_empty_is_none():
     ctx = PostconditionContext(schema_summary="", question="q", attempt=0)
     assert run_postconditions("SELECT 1", None, ctx) is None
     assert run_postconditions("SELECT 1", [], ctx) is None
+
+
+class NonEmpty:
+    """Soft stand-in for "the query returned no rows": a candidate without
+    'WHERE' is treated as empty. Sometimes empty is the right answer, so the
+    objection is soft."""
+
+    code = "non_empty"
+
+    def prompt_section(self) -> str:
+        return ""
+
+    def check(self, query: str, *, context: PostconditionContext):
+        if "WHERE" in query:
+            return None
+        return PostconditionViolation(
+            code=self.code,
+            reason="The query returned no rows.",
+            suggested_hint="Check literals and labels.",
+            soft=True,
+        )
+
+
+def test_soft_violation_is_retried_like_a_hard_one():
+    eng = _engine(["SELECT *", "SELECT * WHERE {}"], [NonEmpty()])
+    r = eng.generate("q")
+    assert r.ok is True and r.retries == 1
+    assert r.query == "SELECT * WHERE {}"
+    assert r.postcondition is None
+    assert "Check literals and labels." in eng.provider.calls[1][1]
+
+
+def test_soft_exhaustion_returns_first_soft_candidate():
+    eng = _engine(["SELECT 1", "SELECT 2", "SELECT 3"], [NonEmpty()], max_retries=2)
+    r = eng.generate("q")
+    assert r.ok is True and r.error == ""
+    assert r.retries == 2  # the whole budget was spent first
+    assert r.query == "SELECT 1"  # the first candidate, not the last
+    assert r.postcondition is not None and r.postcondition.soft
+    assert r.postcondition.code == "non_empty"
+
+
+def test_soft_fallback_survives_a_later_invalid_candidate():
+    eng = _engine(["SELECT 1", "nonsense"], [NonEmpty()], max_retries=1)
+    r = eng.generate("q")
+    assert r.ok is True and r.query == "SELECT 1"
+    assert r.validation is not None and r.validation.ok
+
+
+def test_soft_fallback_survives_a_later_hard_violation():
+    # Soft-only first candidate (has LIMIT, no WHERE); the second breaks the
+    # hard invariant. The soft candidate is the one worth returning.
+    eng = _engine(["SELECT * LIMIT 1", "SELECT *"], [NeedsLimit(), NonEmpty()], max_retries=1)
+    r = eng.generate("q")
+    assert r.ok is True and r.query == "SELECT * LIMIT 1"
+    assert r.postcondition is not None and r.postcondition.code == "non_empty"
+
+
+def test_hard_violation_still_fails_closed_beside_a_soft_check():
+    # Every candidate breaks the hard rule first (first violation wins), so
+    # none is ever soft-only: no fallback, fail closed as before.
+    eng = _engine(["SELECT *"], [NeedsLimit(), NonEmpty()], max_retries=2)
+    r = eng.generate("q")
+    assert r.ok is False and r.query == ""
+    assert r.postcondition is not None and r.postcondition.code == "needs_limit"
+
+
+def test_soft_fallback_still_passes_the_guardrail_gate():
+    eng = _engine(["SELECT 1"], [NonEmpty()], allow=False, max_retries=1)
+    r = eng.generate("q")
+    assert r.ok is False and r.query == ""
+    assert "tenant scope" in r.error
+
+
+def test_violation_is_hard_by_default():
+    assert PostconditionViolation(reason="r", suggested_hint="h").soft is False
+
+
+def test_hard_check_after_a_soft_one_still_applies():
+    # Regression: the soft check used to short-circuit the run, so a query
+    # breaking the hard rule listed after it came back ok as a soft fallback.
+    eng = _engine(["SELECT *"], [NonEmpty(), NeedsLimit()], max_retries=2)
+    r = eng.generate("q")
+    assert r.ok is False and r.query == ""
+    assert r.postcondition is not None and r.postcondition.code == "needs_limit"
+
+
+def test_soft_then_hard_reports_the_hard_correction():
+    eng = _engine(["SELECT *", "SELECT * LIMIT 1 WHERE {}"], [NonEmpty(), NeedsLimit()])
+    r = eng.generate("q")
+    assert r.ok is True and r.retries == 1
+    assert "Add LIMIT 100." in eng.provider.calls[1][1]  # the hard hint, not the soft one
+
+
+def test_raising_check_after_a_soft_one_still_fails_closed():
+    class Boom:
+        code = "boom"
+
+        def prompt_section(self):
+            return ""
+
+        def check(self, query, *, context):
+            raise ValueError("kaboom")
+
+    eng = _engine(["SELECT 1"], [NonEmpty(), Boom()], max_retries=1)
+    r = eng.generate("q")
+    assert r.ok is False and r.query == ""
+    assert r.postcondition is not None and r.postcondition.code == "boom_error"
+
+
+def test_soft_fallback_with_no_retries():
+    eng = _engine(["SELECT 1"], [NonEmpty()], max_retries=0)
+    r = eng.generate("q")
+    assert r.ok is True and r.retries == 0 and r.query == "SELECT 1"
+    assert r.postcondition is not None and r.postcondition.soft
+
+
+def test_clean_last_candidate_beats_the_soft_fallback():
+    eng = _engine(["SELECT 1 LIMIT 1", "SELECT * LIMIT 1 WHERE {}"], [NeedsLimit(), NonEmpty()])
+    r = eng.generate("q")
+    assert r.ok is True and r.query == "SELECT * LIMIT 1 WHERE {}"
+    assert r.postcondition is None
+
+
+def test_run_postconditions_prefers_hard_over_earlier_soft():
+    ctx = PostconditionContext(schema_summary="", question="q", attempt=0)
+    v = run_postconditions("SELECT *", [NonEmpty(), NeedsLimit()], ctx)
+    assert v is not None and v.code == "needs_limit" and not v.soft
+    v = run_postconditions("SELECT * LIMIT 1", [NonEmpty(), NeedsLimit()], ctx)
+    assert v is not None and v.code == "non_empty" and v.soft