attune-verify 0.1.0__tar.gz

attune_verify-0.1.0/LICENSE

@@ -0,0 +1,17 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+Copyright 2026 Smart AI Memory
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

attune_verify-0.1.0/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.4
+Name: attune-verify
+Version: 0.1.0
+Summary: Generation fact-checker for the attune-* family. Verifies named entities in LLM output actually exist — imports import, CLI flags are real, links resolve, counts match source.
+Author-email: Patrick Roebuck <admin@smartaimemory.com>
+License:                                  Apache License
+                                   Version 2.0, January 2004
+                                http://www.apache.org/licenses/
+        
+        Copyright 2026 Smart AI Memory
+        
+        Licensed under the Apache License, Version 2.0 (the "License");
+        you may not use this file except in compliance with the License.
+        You may obtain a copy of the License at
+        
+            http://www.apache.org/licenses/LICENSE-2.0
+        
+        Unless required by applicable law or agreed to in writing, software
+        distributed under the License is distributed on an "AS IS" BASIS,
+        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        See the License for the specific language governing permissions and
+        limitations under the License.
+        
+Project-URL: Homepage, https://github.com/Smart-AI-Memory/attune-verify
+Project-URL: Repository, https://github.com/Smart-AI-Memory/attune-verify
+Keywords: verification,fact-checking,hallucination,llm,attune,grounding,faithfulness
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: rag
+Requires-Dist: attune-rag>=0.2.0; extra == "rag"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
+Requires-Dist: black>=24.0; extra == "dev"
+Requires-Dist: build>=1.0; extra == "dev"
+Requires-Dist: twine>=5.0; extra == "dev"
+Dynamic: license-file
+
+# attune-verify
+
+Generation fact-checker for the attune-\* family. Verifies named entities
+in LLM-generated content actually exist — imports import, CLI flags are real,
+links resolve, counts match source — so hallucinations that pass unit tests
+are caught before they reach a reader.
+
+## Install
+
+```bash
+pip install attune-verify
+```
+
+With the optional LLM semantic layer (requires attune-rag):
+
+```bash
+pip install 'attune-verify[rag]'
+```
+
+## Quick start
+
+```python
+from attune_verify import verify, VerifyContext
+from pathlib import Path
+
+ctx = VerifyContext(
+    project_root=Path("."),
+    allowed_help_cmds=frozenset(["attune"]),
+)
+result = verify(generated_content, ctx)
+if not result.ok:
+    for f in result.findings:
+        print(f"{f.kind}: {f.detail}")
+```
+
+## Part of the attune family
+
+- **attune-rag** grounds generation in accurate retrieved sources (input-side)
+- **attune-verify** checks that named entities in the output actually exist (output-side)
+
+Together they bracket generation: rag verifies *"is this claim supported?"*;
+verify checks *"does this named thing exist?"*
+
+## Status
+
+Pre-alpha — spec complete, implementation in progress.
+
+## License
+
+Apache 2.0

attune_verify-0.1.0/README.md

@@ -0,0 +1,50 @@
+# attune-verify
+
+Generation fact-checker for the attune-\* family. Verifies named entities
+in LLM-generated content actually exist — imports import, CLI flags are real,
+links resolve, counts match source — so hallucinations that pass unit tests
+are caught before they reach a reader.
+
+## Install
+
+```bash
+pip install attune-verify
+```
+
+With the optional LLM semantic layer (requires attune-rag):
+
+```bash
+pip install 'attune-verify[rag]'
+```
+
+## Quick start
+
+```python
+from attune_verify import verify, VerifyContext
+from pathlib import Path
+
+ctx = VerifyContext(
+    project_root=Path("."),
+    allowed_help_cmds=frozenset(["attune"]),
+)
+result = verify(generated_content, ctx)
+if not result.ok:
+    for f in result.findings:
+        print(f"{f.kind}: {f.detail}")
+```
+
+## Part of the attune family
+
+- **attune-rag** grounds generation in accurate retrieved sources (input-side)
+- **attune-verify** checks that named entities in the output actually exist (output-side)
+
+Together they bracket generation: rag verifies *"is this claim supported?"*;
+verify checks *"does this named thing exist?"*
+
+## Status
+
+Pre-alpha — spec complete, implementation in progress.
+
+## License
+
+Apache 2.0

attune_verify-0.1.0/pyproject.toml

@@ -0,0 +1,70 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "attune-verify"
+version = "0.1.0"
+description = "Generation fact-checker for the attune-* family. Verifies named entities in LLM output actually exist — imports import, CLI flags are real, links resolve, counts match source."
+readme = {file = "README.md", content-type = "text/markdown"}
+requires-python = ">=3.10"
+license = {file = "LICENSE"}
+authors = [
+    {name = "Patrick Roebuck", email = "admin@smartaimemory.com"}
+]
+keywords = [
+    "verification", "fact-checking", "hallucination",
+    "llm", "attune", "grounding", "faithfulness",
+]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = []
+
+[project.optional-dependencies]
+rag = ["attune-rag>=0.2.0"]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.0",
+    "ruff>=0.4.0",
+    "black>=24.0",
+    "build>=1.0",
+    "twine>=5.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Smart-AI-Memory/attune-verify"
+Repository = "https://github.com/Smart-AI-Memory/attune-verify"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "B", "I"]
+ignore = ["BLE001"]
+
+[tool.black]
+line-length = 100
+target-version = ["py310", "py311", "py312", "py313"]
+
+[tool.coverage.run]
+source = ["attune_verify"]
+omit = ["tests/*"]

attune_verify-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

attune_verify-0.1.0/src/attune_verify/__init__.py

@@ -0,0 +1,34 @@
+"""attune-verify — generation fact-checker.
+
+Verifies named entities in LLM-generated content actually exist:
+imports import, CLI flags are real, links resolve, counts match source.
+
+Public API::
+
+    from attune_verify import verify, VerifyContext, VerifyResult
+    from attune_verify import Finding, FindingKind
+    from attune_verify import VerificationError, raise_if_failed
+    from attune_verify.semantic.protocol import Judge, SemanticVerdict
+"""
+from __future__ import annotations
+
+from attune_verify.context import VerifyContext
+from attune_verify.result import (
+    Finding,
+    FindingKind,
+    VerificationError,
+    VerifyResult,
+    raise_if_failed,
+)
+from attune_verify._verify import verify
+
+__version__ = "0.1.0"
+__all__ = [
+    "verify",
+    "VerifyContext",
+    "VerifyResult",
+    "Finding",
+    "FindingKind",
+    "VerificationError",
+    "raise_if_failed",
+]

attune_verify-0.1.0/src/attune_verify/_extract.py

@@ -0,0 +1,79 @@
+"""Shared extraction utilities: pull code fences, links, numeric claims."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+
+@dataclass
+class CodeFence:
+    """A fenced code block extracted from markdown."""
+    language: str
+    content: str
+    line: Optional[int] = None
+
+
+@dataclass
+class MarkdownLink:
+    """A markdown link extracted from content."""
+    text: str
+    target: str
+    line: Optional[int] = None
+
+
+@dataclass
+class NumericClaim:
+    """A numeric claim extracted from content."""
+    value: int
+    context: str  # surrounding text
+    line: Optional[int] = None
+
+
+_FENCE_RE = re.compile(
+    r"^```(\w*)\n(.*?)^```",
+    re.MULTILINE | re.DOTALL,
+)
+_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
+_NUM_RE = re.compile(r"\b(\d{2,})\b")  # 2+ digit numbers (skip single digits)
+
+
+def extract_code_fences(content: str) -> List[CodeFence]:
+    """Extract all fenced code blocks from markdown content."""
+    fences = []
+    for match in _FENCE_RE.finditer(content):
+        line = content[: match.start()].count("\n") + 1
+        fences.append(CodeFence(
+            language=match.group(1) or "text",
+            content=match.group(2),
+            line=line,
+        ))
+    return fences
+
+
+def extract_links(content: str) -> List[MarkdownLink]:
+    """Extract all markdown links from content."""
+    links = []
+    for match in _LINK_RE.finditer(content):
+        line = content[: match.start()].count("\n") + 1
+        links.append(MarkdownLink(
+            text=match.group(1),
+            target=match.group(2),
+            line=line,
+        ))
+    return links
+
+
+def extract_numeric_claims(content: str) -> List[NumericClaim]:
+    """Extract numeric claims (2+ digit numbers) with surrounding context."""
+    claims = []
+    for match in _NUM_RE.finditer(content):
+        line = content[: match.start()].count("\n") + 1
+        start = max(0, match.start() - 40)
+        end = min(len(content), match.end() + 40)
+        claims.append(NumericClaim(
+            value=int(match.group(1)),
+            context=content[start:end].replace("\n", " "),
+            line=line,
+        ))
+    return claims

attune_verify-0.1.0/src/attune_verify/_verify.py

@@ -0,0 +1,142 @@
+"""Core verify() orchestration — runs all checkers and the semantic layer."""
+from __future__ import annotations
+
+import logging
+from typing import List
+
+from attune_verify._extract import (
+    extract_code_fences,
+    extract_links,
+    extract_numeric_claims,
+)
+from attune_verify.checkers.counts import check_counts
+from attune_verify.checkers.flags import check_flags
+from attune_verify.checkers.imports import check_imports
+from attune_verify.checkers.links import check_links
+from attune_verify.context import VerifyContext
+from attune_verify.result import Finding, FindingKind, VerifyResult
+
+logger = logging.getLogger(__name__)
+
+
+def verify(content: str, context: VerifyContext) -> VerifyResult:
+    """Run deterministic checkers and optional semantic layer.
+
+    Deterministic checkers always run and are independent — a failure in
+    one does not abort the others. The semantic layer runs only when
+    context.semantic is True and a judge is available.
+
+    Args:
+        content: LLM-generated content to verify.
+        context: Declared truth boundaries (project root, env, commands,
+            count sources, optional judge).
+
+    Returns:
+        VerifyResult with all findings. Never raises on findings — use
+        raise_if_failed(result) for a hard gate.
+    """
+    result = VerifyResult()
+
+    # --- Deterministic checkers ---
+    _run_checker(result, "imports", _check_imports, content, context)
+    _run_checker(result, "flags", _check_flags, content, context)
+    _run_checker(result, "links", _check_links, content, context)
+    _run_checker(result, "counts", _check_counts, content, context)
+
+    # --- Semantic layer (opt-in) ---
+    if context.semantic:
+        _run_semantic(result, content, context)
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Per-checker wrappers — each catches exceptions and surfaces as a warning
+# ---------------------------------------------------------------------------
+
+def _check_imports(content: str, context: VerifyContext) -> List[Finding]:
+    fences = extract_code_fences(content)
+    return check_imports(fences, env_python=context.env_python)
+
+
+def _check_flags(content: str, context: VerifyContext) -> List[Finding]:
+    return check_flags(
+        content,
+        help_commands=context.help_commands,
+        allowed_help_cmds=context.allowed_help_cmds,
+    )
+
+
+def _check_links(content: str, context: VerifyContext) -> List[Finding]:
+    links = extract_links(content)
+    return check_links(links, project_root=context.project_root)
+
+
+def _check_counts(content: str, context: VerifyContext) -> List[Finding]:
+    claims = extract_numeric_claims(content)
+    return check_counts(claims, count_sources=context.count_sources)
+
+
+def _run_checker(
+    result: VerifyResult,
+    name: str,
+    fn: object,
+    content: str,
+    context: VerifyContext,
+) -> None:
+    """Run one checker, surfacing any internal exception as a warning."""
+    try:
+        findings = fn(content, context)  # type: ignore[operator]
+        result.findings.extend(findings)
+        result.checked.append(name)
+    except Exception as exc:  # noqa: BLE001
+        # INTENTIONAL: individual checker failures must not abort the run.
+        logger.exception("checker '%s' raised: %s", name, exc)
+        result.findings.append(Finding(
+            kind=FindingKind.UNRESOLVED_IMPORT,  # closest kind for infra error
+            detail=f"Checker '{name}' failed: {exc}",
+            evidence="",
+            severity="warning",
+        ))
+
+
+def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> None:
+    """Run the semantic layer if a judge is available."""
+    from attune_verify.semantic.protocol import Judge  # noqa: PLC0415
+
+    if context.judge is None or not isinstance(context.judge, Judge):
+        result.findings.append(Finding(
+            kind=FindingKind.SEMANTIC,
+            detail=(
+                "Semantic layer requested (context.semantic=True) "
+                "but no judge was provided in VerifyContext.judge"
+            ),
+            evidence="",
+            severity="warning",
+        ))
+        return
+
+    try:
+        verdict = context.judge.score(
+            query="Verify this generated content for faithfulness",
+            answer=content,
+            passages=content,
+        )
+        result.semantic_ran = True
+        if not verdict.faithful:
+            for issue in verdict.issues:
+                result.findings.append(Finding(
+                    kind=FindingKind.SEMANTIC,
+                    detail=issue,
+                    evidence="",
+                    severity="error",
+                ))
+    except Exception as exc:  # noqa: BLE001
+        # INTENTIONAL: semantic layer is opt-in; failures degrade gracefully.
+        logger.exception("semantic judge raised: %s", exc)
+        result.findings.append(Finding(
+            kind=FindingKind.SEMANTIC,
+            detail=f"Semantic judge failed: {exc}",
+            evidence="",
+            severity="warning",
+        ))

attune_verify-0.1.0/src/attune_verify/checkers/__init__.py

@@ -0,0 +1 @@
+"""Deterministic entity-resolution checkers for attune-verify."""

attune_verify-0.1.0/src/attune_verify/checkers/counts.py

@@ -0,0 +1,68 @@
+"""Count checker — verifies numeric claims match caller-supplied sources."""
+from __future__ import annotations
+
+from typing import Callable, Dict, List, Union
+
+from attune_verify._extract import NumericClaim
+from attune_verify.result import Finding, FindingKind
+
+
+def check_counts(
+    claims: List[NumericClaim],
+    count_sources: Dict[str, Union[int, Callable[[], int]]],
+) -> List[Finding]:
+    """Verify numeric claims match count_sources values.
+
+    Counts cannot be inferred — the caller must supply them. Any numeric
+    claim in the content is matched against count_sources by value. Claims
+    with no matching source entry are flagged as warnings (unverifiable),
+    not errors.
+
+    Args:
+        claims: Numeric claims extracted from generated content.
+        count_sources: Expected values keyed by label/description.
+            Values may be plain ints or zero-argument callables.
+
+    Returns:
+        List of findings for mismatched or unverifiable counts.
+    """
+    if not count_sources:
+        return []
+
+    resolved_sources: Dict[str, int] = {}
+    for label, value in count_sources.items():
+        resolved_sources[label] = value() if callable(value) else value
+
+    expected_values = set(resolved_sources.values())
+    findings: List[Finding] = []
+
+    for claim in claims:
+        if claim.value not in expected_values:
+            # Check if it could be a mismatch against a named source
+            close_label = _find_close_label(claim.context, resolved_sources)
+            if close_label is not None:
+                expected = resolved_sources[close_label]
+                findings.append(Finding(
+                    kind=FindingKind.COUNT_MISMATCH,
+                    detail=(
+                        f"Count {claim.value} doesn't match "
+                        f"'{close_label}' (expected {expected})"
+                    ),
+                    evidence=claim.context,
+                    location=f"line {claim.line}" if claim.line else None,
+                    severity="error",
+                ))
+    return findings
+
+
+def _find_close_label(
+    context: str,
+    sources: Dict[str, int],
+) -> str | None:
+    """Find a source label whose keywords appear in the claim's context."""
+    context_lower = context.lower()
+    for label in sources:
+        words = label.lower().split()
+        if any(w in context_lower for w in words if len(w) > 3):
+            return label
+    return None

attune_verify-0.1.0/src/attune_verify/checkers/flags.py

@@ -0,0 +1,85 @@
+"""Flag checker — verifies CLI flags referenced in content exist in --help."""
+from __future__ import annotations
+
+import re
+import subprocess
+from typing import Dict, FrozenSet, List
+
+from attune_verify.result import Finding, FindingKind
+
+_FLAG_RE = re.compile(r"`(--[\w-]+)`")
+
+
+def check_flags(
+    content: str,
+    help_commands: Dict[str, str],
+    allowed_help_cmds: FrozenSet[str],
+) -> List[Finding]:
+    """Verify flags referenced in content exist in command --help output.
+
+    Security: only invokes --help for commands in allowed_help_cmds.
+    A flag for an unknown command yields a warning, not a silent pass.
+
+    Args:
+        content: Generated content to scan for flag references.
+        help_commands: Pre-captured --help text keyed by command name.
+        allowed_help_cmds: Commands safe to invoke at runtime.
+
+    Returns:
+        List of findings for unverifiable or unknown flags.
+    """
+    findings: List[Finding] = []
+    # Find patterns like "`--flag`" or "`command --flag`"
+    for match in _FLAG_RE.finditer(content):
+        flag = match.group(1)
+        surrounding = content[max(0, match.start() - 30): match.start()]
+        cmd = _guess_command(surrounding)
+        help_text = _get_help(cmd, help_commands, allowed_help_cmds)
+        if help_text is None:
+            findings.append(Finding(
+                kind=FindingKind.UNKNOWN_FLAG,
+                detail=(
+                    f"Flag '{flag}' could not be verified "
+                    f"(command '{cmd}' not in allowed_help_cmds)"
+                ),
+                evidence=match.group(0),
+                severity="warning",
+            ))
+        elif flag not in help_text:
+            findings.append(Finding(
+                kind=FindingKind.UNKNOWN_FLAG,
+                detail=f"Flag '{flag}' not found in '{cmd} --help'",
+                evidence=match.group(0),
+                severity="error",
+            ))
+    return findings
+
+
+def _guess_command(preceding: str) -> str:
+    """Heuristically extract the command name preceding a flag."""
+    words = preceding.strip().split()
+    for word in reversed(words):
+        cleaned = word.strip("`")
+        if cleaned and not cleaned.startswith("-"):
+            return cleaned
+    return "unknown"
+
+
+def _get_help(
+    cmd: str,
+    help_commands: Dict[str, str],
+    allowed_help_cmds: FrozenSet[str],
+) -> str | None:
+    """Return help text or None if the command cannot be introspected."""
+    if cmd in help_commands:
+        return help_commands[cmd]
+    if cmd in allowed_help_cmds:
+        result = subprocess.run(
+            [cmd, "--help"],
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            timeout=10,
+        )
+        return result.stdout + result.stderr
+    return None