ummaya 0.2.4 → 0.2.6

package/src/ummaya/_canonical/__init__.py

@@ -12,6 +12,8 @@ depending on the source-tree layout:
 * ``checklist_manifest.yaml`` — 50-row plugin validation checklist.
   Loaded by :mod:`ummaya.plugins.checks.framework` and the
   ``ummaya-plugin-validate`` CLI.
+* ``document-tools.schema.json`` — Public AX document harness tool
+  contract. Loaded by :mod:`ummaya.tools.documents.contracts`.
 
 The mapping is in pyproject.toml under
 ``[tool.hatch.build.targets.wheel.force-include]``.

package/src/ummaya/_canonical/baselines.yaml

@@ -0,0 +1,113 @@
+version: 1
+catalog_id: public-form-conformance-baselines-v1
+source_policy: offline_fixtures_only
+live_network_allowed: false
+baselines:
+  - template_id: birth-benefit-application-hwpx
+    schema_id: birth-benefit-application-v1
+    format: hwpx
+    authoritative_standard: KS X 6101 OWPML/HWPX
+    authority_refs:
+      - specs/2803-document-production-hardening/spec.md#FR-002
+      - specs/2803-document-production-hardening/spec.md#FR-012
+    supports_conformance: true
+    expected_page_count: 2
+    required_fields:
+      - field_id: applicant_name
+        label: Applicant name
+        path: /body/section[1]/field[applicant_name]
+      - field_id: child_birth_date
+        label: Child birth date
+        path: /body/section[1]/field[child_birth_date]
+    protected_text:
+      - text: Birth benefit application
+        anchor: /body/section[1]/p[1]
+      - text: Required attachments
+        anchor: /body/section[2]/p[1]
+    required_labels:
+      - text: Applicant
+        anchor: /body/section[1]/table[1]/cell[1,1]
+      - text: Name
+        anchor: /body/section[1]/table[1]/cell[1,2]
+      - text: Child
+        anchor: /body/section[1]/table[1]/cell[2,1]
+      - text: Date of birth
+        anchor: /body/section[1]/table[1]/cell[2,2]
+    table_geometries:
+      - table_id: applicant-table
+        anchor: /body/section[1]/table[1]
+        rows: 2
+        columns: 2
+    signature_regions:
+      - text: Applicant signature or seal
+        anchor: /body/section[1]/p[2]
+    metadata_exact_matches:
+      page_count: 2
+      margin_top_mm: 20
+      margin_bottom_mm: 15
+  - template_id: civil-form-docx
+    schema_id: civil-form-ooxml-v1
+    format: docx
+    authoritative_standard: ECMA-376 Office Open XML
+    authority_refs:
+      - specs/2803-document-production-hardening/spec.md#FR-012
+    supports_conformance: true
+    required_fields: []
+    protected_text: []
+    required_labels: []
+    table_geometries: []
+    signature_regions: []
+    metadata_exact_matches: {}
+  - template_id: civil-ledger-xlsx
+    schema_id: civil-ledger-ooxml-v1
+    format: xlsx
+    authoritative_standard: ECMA-376 Office Open XML
+    authority_refs:
+      - specs/2803-document-production-hardening/spec.md#FR-012
+    supports_conformance: true
+    required_fields: []
+    protected_text: []
+    required_labels: []
+    table_geometries: []
+    signature_regions: []
+    metadata_exact_matches: {}
+  - template_id: civil-briefing-pptx
+    schema_id: civil-briefing-ooxml-v1
+    format: pptx
+    authoritative_standard: ECMA-376 Office Open XML
+    authority_refs:
+      - specs/2803-document-production-hardening/spec.md#FR-012
+    supports_conformance: true
+    required_fields: []
+    protected_text: []
+    required_labels: []
+    table_geometries: []
+    signature_regions: []
+    metadata_exact_matches: {}
+  - template_id: fillable-permit-pdf
+    schema_id: fillable-permit-pdf-v1
+    format: pdf
+    authoritative_standard: PDF AcroForm and rendered appearance
+    authority_refs:
+      - specs/2803-document-production-hardening/spec.md#FR-014
+    supports_conformance: true
+    required_fields: []
+    protected_text: []
+    required_labels: []
+    table_geometries: []
+    signature_regions: []
+    metadata_exact_matches: {}
+  - template_id: legacy-hwp-readonly
+    schema_id: legacy-hwp-readonly-v1
+    format: hwp
+    authoritative_standard: HWP binary read-only boundary
+    authority_refs:
+      - specs/2803-document-production-hardening/spec.md#FR-017
+    supports_conformance: false
+    unsupported_reason: HWP binary write is blocked in this epic; conformance-ready output requires HWPX or another promoted editable derivative.
+    required_fields: []
+    protected_text: []
+    required_labels: []
+    table_geometries: []
+    signature_regions: []
+    metadata_exact_matches: {}

package/src/ummaya/engine/engine.py

@@ -8,7 +8,6 @@ the standalone ``query()`` async generator.
 
 from __future__ import annotations
 
-import json
 import logging
 from collections.abc import AsyncIterator
 from typing import TYPE_CHECKING
@@ -20,9 +19,13 @@ from ummaya.engine.models import QueryContext, QueryState, SessionBudget
 from ummaya.engine.query import query
 from ummaya.llm.client import LLMClient
 from ummaya.llm.models import ChatMessage
-from ummaya.tools.errors import ToolNotFoundError
 from ummaya.tools.executor import ToolExecutor
 from ummaya.tools.registry import ToolRegistry
+from ummaya.tools.routing import (
+    RouteDecisionService,
+    build_available_adapters_projection,
+    selected_concrete_adapter_tools,
+)
 
 if TYPE_CHECKING:
     from ummaya.permissions.models import SessionContext
@@ -31,66 +34,6 @@ logger = logging.getLogger(__name__)
 
 _INTERNAL_CONTEXT_TOOL_IDS = frozenset({"find", "locate", "check", "send", "search_tools"})
 
-_LOCATION_DEPENDENT_SCHEMA_KEYS = frozenset(
-    {
-        "adm_cd",
-        "admcd",
-        "admin_code",
-        "administrative_code",
-        "latitude",
-        "lat",
-        "longitude",
-        "lon",
-        "lng",
-        "nx",
-        "ny",
-        "region_cd",
-        "region_code",
-    }
-)
-
-
-def _schema_requires_location_resolution(
-    input_schema_json: object,
-    required_params: object,
-) -> bool:
-    """Return True when an adapter schema needs prior locate output."""
-
-    return _contains_location_dependent_key(
-        required_params
-    ) or _schema_required_fields_contain_location_key(input_schema_json)
-
-
-def _schema_required_fields_contain_location_key(value: object) -> bool:
-    """Return True when JSON Schema required fields demand locate-derived data."""
-
-    if isinstance(value, dict):
-        required = value.get("required")
-        if _contains_location_dependent_key(required):
-            return True
-        return any(
-            _schema_required_fields_contain_location_key(nested) for nested in value.values()
-        )
-    if isinstance(value, list):
-        return any(_schema_required_fields_contain_location_key(item) for item in value)
-    return False
-
-
-def _contains_location_dependent_key(value: object) -> bool:
-    """Recursively detect coordinate/admin-code schema fields."""
-
-    if isinstance(value, dict):
-        for key, nested in value.items():
-            if str(key).lower() in _LOCATION_DEPENDENT_SCHEMA_KEYS:
-                return True
-            if _contains_location_dependent_key(nested):
-                return True
-    elif isinstance(value, list):
-        return any(_contains_location_dependent_key(item) for item in value)
-    elif isinstance(value, str):
-        return value.lower() in _LOCATION_DEPENDENT_SCHEMA_KEYS
-    return False
-
 
 class QueryEngine:
     """Per-session orchestrator for the UMMAYA query engine.
@@ -296,85 +239,39 @@ class QueryEngine:
         """Build dynamic adapter context and per-turn concrete tool exposure."""
 
         try:
-            from ummaya.tools.search import search  # noqa: PLC0415
-
-            candidates = search(
+            decision = RouteDecisionService(self._tool_registry).select_adapters(
                 user_message,
-                self._tool_registry.bm25_index,
-                self._tool_registry,
                 top_k=15,
+                max_selected=5,
             )
-        except Exception:  # noqa: BLE001
-            logger.exception("available_adapters auto-inject failed")
-            return None, ()
-
-        adapter_lines: list[str] = []
-        selected_tool_ids: list[str] = []
-        primary_find_without_location = False
-        visible_count = 0
-        for candidate in candidates:
-            try:
-                tool = self._tool_registry.find(candidate.tool_id)
-            except ToolNotFoundError:
-                continue
-            if candidate.score <= 0:
-                continue
-            if tool.id in _INTERNAL_CONTEXT_TOOL_IDS:
-                continue
-            primitive = candidate.primitive if isinstance(candidate.primitive, str) else None
-            requires_location = _schema_requires_location_resolution(
-                candidate.input_schema_json,
-                candidate.required_params,
-            )
-            primary_find_without_location = primary_find_without_location or (
-                visible_count == 0 and primitive == "find" and not requires_location
+            concrete_tools = selected_concrete_adapter_tools(
+                decision,
+                self._tool_registry,
+                exclude_tool_ids=_INTERNAL_CONTEXT_TOOL_IDS,
+                max_tools=5,
             )
-            if visible_count > 0 and primary_find_without_location and requires_location:
-                continue
-            schema_json = json.dumps(
-                candidate.input_schema_json,
-                ensure_ascii=False,
-                sort_keys=True,
+            tool_ids = tuple(tool.id for tool in concrete_tools)
+            if not tool_ids:
+                return None, ()
+            projection_level = (
+                decision.schema_projection_level if decision.selected_tools else "summary"
             )
-            selected_tool_ids.append(candidate.tool_id)
-            adapter_lines.extend(
-                [
-                    f"- tool_id: {candidate.tool_id}",
-                    f"  primitive: {candidate.primitive}",
-                    f"  description: {candidate.llm_description or tool.name_ko}",
-                    f"  required_params: {candidate.required_params}",
-                    f"  input_schema_json: {schema_json}",
-                    f"  call_hint: {candidate.tool_id}({{...}})",
-                    f"  policy_url: {candidate.real_classification_url or ''}",
-                ]
+            projection = build_available_adapters_projection(
+                decision,
+                self._tool_registry,
+                query=user_message,
+                projection_level=projection_level,
+                max_visible=len(tool_ids),
+                visible_tool_ids=tool_ids,
             )
-            visible_count += 1
-            if visible_count >= 5:
-                break
+        except Exception:  # noqa: BLE001
+            logger.exception("available_adapters auto-inject failed")
+            return None, ()
 
-        if not adapter_lines:
+        if projection.content is None:
             return None, ()
 
-        content = "\n".join(
-            [
-                "<available_adapters>",
-                "Use these adapter candidates for this citizen request. "
-                "The model-facing function name is the concrete tool_id shown "
-                "below when that function is present in tools[]. Call the "
-                "concrete adapter directly with exactly the input_schema_json "
-                "fields. Do not wrap tool_id/params inside a concrete adapter "
-                "call. The root primitives (find, locate, check, send) are "
-                "legacy compatibility wrappers only when a concrete adapter "
-                "function is not loaded. "
-                "Do not call locate just because the citizen text contains a "
-                "city/province name; treat that as the dataset/filter term. "
-                "Call locate only when the selected adapter schema requires "
-                "coordinates, administrative codes, or a place-to-region conversion.",
-                *adapter_lines,
-                "</available_adapters>",
-            ]
-        )
-        return ChatMessage(role="system", content=content), tuple(selected_tool_ids)
+        return ChatMessage(role="system", content=projection.content), tool_ids
 
     def set_permission_session(self, session: SessionContext | None) -> None:
         """Update the permission-pipeline session used for subsequent turns.

package/src/ummaya/evidence/__init__.py

@@ -1,8 +1,20 @@
 # SPDX-License-Identifier: Apache-2.0
 """Evidence Fabric v2 public API."""
 
-from ummaya.evidence.models import EvidenceGate, EvidenceStatus, RunEvidence
-from ummaya.evidence.runner import EvidenceContractError, run_dataset
+from ummaya.evidence.dataset_contract import EvidenceContractError
+from ummaya.evidence.models import (
+    EvidenceGate,
+    EvidenceStatus,
+    RunEvidence,
+)
+from ummaya.evidence.runner import run_dataset
+from ummaya.evidence.source_provenance import (
+    SourceProvenanceDecision,
+    SourceProvenanceLedger,
+    SourceProvenanceRecord,
+    SourceRedactionMetadata,
+    build_source_provenance_record,
+)
 from ummaya.evidence.task_registry import (
     EvidenceDatasetRef,
     EvidenceTask,
@@ -10,6 +22,7 @@ from ummaya.evidence.task_registry import (
     TaskRegistryError,
     load_task_registry,
 )
+from ummaya.evidence.tool_layer_models import ToolLayerEvidenceEvent
 
 __all__ = [
     "EvidenceDatasetRef",
@@ -19,7 +32,13 @@ __all__ = [
     "EvidenceTask",
     "EvidenceTaskRegistry",
     "RunEvidence",
+    "SourceProvenanceDecision",
+    "SourceProvenanceLedger",
+    "SourceProvenanceRecord",
+    "SourceRedactionMetadata",
     "TaskRegistryError",
+    "ToolLayerEvidenceEvent",
+    "build_source_provenance_record",
     "load_task_registry",
     "run_dataset",
 ]

package/src/ummaya/evidence/dataset_contract.py

@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Scenario dataset parsing for Evidence Fabric v2."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import assert_never
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from ummaya.evidence.json_types import JsonObject, JsonValue, parse_json_object
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+DEFAULT_SCENARIO_PATH = _REPO_ROOT / "evidence/scenarios/national_ax_citizen_requests_v1.yaml"
+DEFAULT_TASK_REGISTRY_PATH = _REPO_ROOT / "evidence/registry.yaml"
+DEFAULT_DATASET_REF = "ummaya/national-ax-core@local"
+BANNED_MODEL_VISIBLE_KEYS = frozenset(
+    {
+        "adapter_id",
+        "adapter_ids",
+        "adapter_family",
+        "expected_adapter_id",
+        "tool_id",
+        "tool_ids",
+        "expected_tool_id",
+        "expected_tool_ids",
+        "expected_adapter_family",
+        "expected_route_trace",
+        "route_trace",
+        "route_selection_assertion",
+        "route_selection_assertions",
+        "route_adapter_family",
+        "selected_adapter_family",
+        "selected_adapter_id",
+        "selected_tool",
+        "selected_tool_id",
+        "selected_tool_ids",
+        "selected_tools",
+        "fixture_refs",
+        "fixture_ref",
+        "current_adapter_id",
+        "assertion_events",
+        "assertion_kind",
+        "argument_feasibility",
+        "clarification_expected",
+        "clarification_reason",
+        "correlation_id",
+        "coverage_tags",
+        "evidence_events",
+        "expected_domain",
+        "expected_primitives",
+        "failure_recovery",
+        "manifest_hash",
+        "prompt_manifest_hash",
+        "query_hash",
+        "route_source",
+        "selected_domain",
+        "selected_primitives",
+        "status",
+        "stop_reason",
+        "tool_catalog_hash",
+        "trace_id",
+        "trace_kind",
+    }
+)
+REQUIRED_DOMAINS = frozenset(
+    {
+        "tax",
+        "civil_affairs",
+        "payments",
+        "utilities",
+        "identity",
+        "welfare",
+        "healthcare",
+        "housing",
+        "mobility",
+        "business",
+        "labor",
+        "education",
+        "safety",
+        "immigration",
+        "legal",
+        "personal_data",
+        "public_data",
+    }
+)
+
+
+class EvidenceContractError(ValueError):
+    """Raised when a scenario dataset violates the Evidence Fabric contract."""
+
+
+class ExpectedStep(BaseModel):
+    """One expected public-service loop step in a scenario."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    primitive: str
+    purpose: str
+
+
+class PermissionRequirements(BaseModel):
+    """Permission requirements attached to a citizen scenario."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    identity_assurance: str
+    user_confirmations: tuple[str, ...] = Field(default_factory=tuple)
+    sensitive_data: tuple[str, ...] = Field(default_factory=tuple)
+
+
+class Scenario(BaseModel):
+    """Minimum scenario shape needed by Evidence Fabric v2."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    id: str
+    priority: str = "P2"
+    lifecycle_domain: str
+    request_ko: str
+    request_en: str | None = None
+    agencies_or_infrastructure: tuple[str, ...] = Field(default_factory=tuple)
+    citizen_intent_verbs: tuple[str, ...] = Field(default_factory=tuple)
+    expected_ax_chain: tuple[ExpectedStep, ...]
+    permission_requirements: PermissionRequirements
+    expected_system_behavior: tuple[str, ...] = Field(default_factory=tuple)
+    evaluation_focus: tuple[str, ...] = Field(default_factory=tuple)
+
+
+class ScenarioDataset(BaseModel):
+    """Versioned citizen-demand scenario dataset."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    version: int
+    dataset_id: str
+    source_basis: str | None = None
+    target_system: str | None = None
+    allowed_primitives: tuple[str, ...] = Field(default_factory=tuple)
+    coverage_domains: tuple[str, ...]
+    scenarios: tuple[Scenario, ...]
+
+
+def parse_dataset(path: Path) -> ScenarioDataset:
+    """Parse and validate a scenario dataset file."""
+    raw = _load_yaml_mapping(path)
+    banned = _find_banned_keys(raw)
+    if banned:
+        raise EvidenceContractError(
+            "model-visible scenario dataset contains banned implementation keys: "
+            + ", ".join(banned)
+        )
+    try:
+        return ScenarioDataset.model_validate(raw)
+    except ValidationError as exc:
+        raise EvidenceContractError(str(exc)) from exc
+
+
+def resolve_repo_path(path: Path) -> Path:
+    """Resolve a repository-relative path from the Evidence Fabric root."""
+    return path if path.is_absolute() else _REPO_ROOT / path
+
+
+def _load_yaml_mapping(path: Path) -> JsonObject:
+    if not path.exists():
+        raise EvidenceContractError(f"scenario dataset not found: {path}")
+    loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
+    try:
+        return parse_json_object(loaded)
+    except ValidationError as exc:
+        raise EvidenceContractError(f"scenario dataset must be a JSON mapping: {path}") from exc
+
+
+def _find_banned_keys(value: JsonValue, path: str = "$") -> tuple[str, ...]:
+    match value:
+        case dict():
+            hits: list[str] = []
+            for key, nested in value.items():
+                nested_path = f"{path}.{key}"
+                if key in BANNED_MODEL_VISIBLE_KEYS:
+                    hits.append(nested_path)
+                hits.extend(_find_banned_keys(nested, nested_path))
+            return tuple(hits)
+        case list():
+            hits = []
+            for index, nested in enumerate(value):
+                hits.extend(_find_banned_keys(nested, f"{path}[{index}]"))
+            return tuple(hits)
+        case str() | int() | float() | bool() | None:
+            return ()
+        case unreachable:
+            assert_never(unreachable)

package/src/ummaya/evidence/document_authoring_cases.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+DocumentAuthoringScenarioClass = Literal[
+    "public_form_completion",
+    "narrative_authoring",
+    "unsupported_plausible_writing",
+    "protected_field",
+    "direct_hwp_path",
+    "render_comparison",
+]
+DocumentAuthoringStatus = Literal["ready_for_review", "needs_input", "blocked"]
+DocumentHwpDirectWriteState = Literal["blocked", "promoted", "not_applicable"]
+
+
+class DocumentAuthoringCase(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    case_id: str
+    scenario_class: DocumentAuthoringScenarioClass
+    correlation_id: str
+    expected_status: DocumentAuthoringStatus
+    requires_socratic_loop: bool
+    requires_user_approval: bool
+    mutation_allowed: bool
+    render_comparison_required: bool
+    hwp_direct_write_state: DocumentHwpDirectWriteState = "not_applicable"
+    fixture_id: str | None = None
+    evidence_ref: str