ummaya 0.2.4 → 0.2.6

package/src/ummaya/tools/documents/evaluation.py

@@ -0,0 +1,291 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Candidate-engine evaluation loop for the Public AX document harness."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal, cast
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from ummaya.tools.documents.capability import (
+    CapabilityOperation,
+    DocumentFormat,
+    FormatCapabilityProfile,
+    PromotionOperation,
+    evaluate_capability_promotion,
+)
+from ummaya.tools.documents.scorecard import CapabilityScorecard
+
+GateState = Literal["pass", "fail", "defer"]
+DependencyPermissionBoundary = Literal["document_primitive_only", "not_applicable"]
+DependencyReason = Literal[
+    "dependency_gate_failed",
+    "dependency_gate_deferred",
+    "license_gate_failed",
+    "license_gate_deferred",
+]
+CandidateDecisionReason = str
+DataGoKrUsageScope = Literal[
+    "semantic_structure_evaluation",
+    "table_extraction_evaluation",
+    "image_reference_evaluation",
+    "summary_rewrite_evaluation",
+]
+SemanticMetricComponent = Literal[
+    "paragraph_block_f1",
+    "table_cell_f1",
+    "image_reference_f1",
+    "metadata_exact_match",
+]
+
+
+class CandidateLicenseGate(BaseModel):
+    """License review gate for one candidate engine."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    spdx: str = Field(min_length=1)
+    gate: GateState
+    notes: str = Field(min_length=1)
+
+
+class CandidateDependencyGate(BaseModel):
+    """Runtime dependency review gate for one candidate engine."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    runtime_dependency: bool
+    gate: GateState
+    notes: str = Field(min_length=1)
+    requires_adr: bool = False
+    adr_ref: str | None = None
+    permission_boundary: DependencyPermissionBoundary = "not_applicable"
+    local_only_execution: bool = True
+    package_ref: str | None = None
+
+    @model_validator(mode="after")
+    def _enforce_runtime_bridge_gate(self) -> CandidateDependencyGate:
+        if self.gate == "pass" and not self.local_only_execution:
+            raise ValueError("local_only_execution must be true for passed dependency gates")
+        if not self.requires_adr:
+            return self
+        if self.adr_ref is None:
+            raise ValueError("requires_adr dependency gates require adr_ref")
+        if not self.adr_ref.startswith("docs/adr/ADR-"):
+            raise ValueError("adr_ref must point at docs/adr/ADR-*.md")
+        if self.permission_boundary != "document_primitive_only":
+            raise ValueError(
+                "requires_adr document bridges must use document_primitive_only permission"
+            )
+        if self.package_ref is None:
+            raise ValueError("requires_adr dependency gates require package_ref")
+        return self
+
+
+class CandidateProfile(BaseModel):
+    """Fixture-backed candidate profile evaluated by the promotion loop."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    format: DocumentFormat
+    engine_id: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9_.-]+$")
+    supported_operations: tuple[CapabilityOperation, ...] = Field(min_length=1)
+    evaluate_operations: tuple[PromotionOperation, ...] = Field(min_length=1)
+    license: CandidateLicenseGate
+    dependency: CandidateDependencyGate
+    scorecard: CapabilityScorecard
+    evidence_refs: tuple[str, ...]
+    decision_note: str = Field(min_length=1)
+    operation_notes: dict[PromotionOperation, str] = Field(default_factory=dict)
+
+    @field_validator("supported_operations", "evaluate_operations")
+    @classmethod
+    def _operations_are_unique(cls, operations: tuple[str, ...]) -> tuple[str, ...]:
+        if len(set(operations)) != len(operations):
+            raise ValueError("operations must be unique")
+        return operations
+
+    @field_validator("evidence_refs")
+    @classmethod
+    def _evidence_is_offline(cls, evidence_refs: tuple[str, ...]) -> tuple[str, ...]:
+        if not evidence_refs:
+            raise ValueError("evidence_refs must not be empty")
+        if any(ref.startswith("live:") for ref in evidence_refs):
+            raise ValueError("live evidence refs are forbidden")
+        return evidence_refs
+
+
+class CandidateProfileManifest(BaseModel):
+    """Offline manifest of candidate engine profiles."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    version: Literal[1]
+    run_id: str = Field(min_length=1)
+    created_at: str = Field(min_length=1)
+    source_policy: Literal["offline_fixtures_only"]
+    live_network_allowed: Literal[False]
+    profiles: tuple[CandidateProfile, ...] = Field(min_length=1)
+
+
+class CandidateDecision(BaseModel):
+    """One operation-level candidate evaluation decision."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    format: DocumentFormat
+    engine_id: str
+    operation: PromotionOperation
+    promoted: bool
+    score: int
+    required_score: int
+    reasons: tuple[CandidateDecisionReason, ...]
+    dependency_gate_passed: bool
+    license_gate_passed: bool
+    decision_note: str
+    evidence_refs: tuple[str, ...]
+
+
+class CandidateEvaluationRun(BaseModel):
+    """Result of evaluating every candidate operation in a manifest."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    run_id: str
+    live_network_allowed: Literal[False]
+    decisions: tuple[CandidateDecision, ...]
+
+
+class DataGoKrMetadataRecord(BaseModel):
+    """One offline metadata record for semantic public-document evaluation."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    record_id: str = Field(min_length=1)
+    provider: str = Field(min_length=1)
+    expected_semantic_fields: tuple[str, ...] = Field(min_length=1)
+    negative_assertion: str = Field(min_length=1)
+
+
+class DataGoKrMetadataSnapshot(BaseModel):
+    """Offline data.go.kr corpus metadata used as evaluation context only."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    version: Literal[1]
+    snapshot_id: str = Field(min_length=1)
+    created_at: str = Field(min_length=1)
+    source_policy: Literal["offline_metadata_snapshot"]
+    live_network_allowed: Literal[False]
+    dataset_name: str = Field(min_length=1)
+    source_ref: str = Field(min_length=1)
+    redistribution_status: Literal["metadata_only"]
+    authoritative_layout_oracle: Literal[False]
+    usage_scope: tuple[DataGoKrUsageScope, ...] = Field(min_length=1)
+    metric_components: tuple[SemanticMetricComponent, ...] = Field(min_length=1)
+    aggregate_threshold: float = Field(ge=0, le=1)
+    records: tuple[DataGoKrMetadataRecord, ...] = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def _require_macro_average_components(self) -> DataGoKrMetadataSnapshot:
+        required: tuple[SemanticMetricComponent, ...] = (
+            "paragraph_block_f1",
+            "table_cell_f1",
+            "image_reference_f1",
+            "metadata_exact_match",
+        )
+        if self.metric_components != required:
+            raise ValueError("metric_components must match the FR-037 macro-average order")
+        return self
+
+
+def load_candidate_profiles(path: Path) -> CandidateProfileManifest:
+    """Load candidate profiles from an offline YAML fixture."""
+    raw = _load_yaml_mapping(path)
+    return CandidateProfileManifest.model_validate(raw)
+
+
+def evaluate_candidate_profiles(
+    manifest: CandidateProfileManifest,
+) -> CandidateEvaluationRun:
+    """Evaluate profile operations through dependency, license, and score gates."""
+    decisions: list[CandidateDecision] = []
+    for profile in manifest.profiles:
+        capability_profile = FormatCapabilityProfile(
+            format=profile.format,
+            engine_id=profile.engine_id,
+            supported_operations=profile.supported_operations,
+            evidence_refs=profile.evidence_refs,
+        )
+        for operation in profile.evaluate_operations:
+            promotion = evaluate_capability_promotion(
+                capability_profile,
+                operation,
+                profile.scorecard,
+            )
+            reasons: list[str] = list(promotion.reasons)
+            dependency_reason = _gate_reason(
+                profile.dependency.gate,
+                failed_reason="dependency_gate_failed",
+                deferred_reason="dependency_gate_deferred",
+            )
+            license_reason = _gate_reason(
+                profile.license.gate,
+                failed_reason="license_gate_failed",
+                deferred_reason="license_gate_deferred",
+            )
+            dependency_gate_passed = dependency_reason is None
+            license_gate_passed = license_reason is None
+            if dependency_reason is not None:
+                reasons.append(dependency_reason)
+            if license_reason is not None:
+                reasons.append(license_reason)
+
+            decisions.append(
+                CandidateDecision(
+                    format=profile.format,
+                    engine_id=profile.engine_id,
+                    operation=operation,
+                    promoted=promotion.promoted and dependency_gate_passed and license_gate_passed,
+                    score=promotion.score,
+                    required_score=promotion.required_score,
+                    reasons=tuple(reasons),
+                    dependency_gate_passed=dependency_gate_passed,
+                    license_gate_passed=license_gate_passed,
+                    decision_note=profile.operation_notes.get(operation, profile.decision_note),
+                    evidence_refs=profile.evidence_refs,
+                )
+            )
+    return CandidateEvaluationRun(
+        run_id=manifest.run_id,
+        live_network_allowed=manifest.live_network_allowed,
+        decisions=tuple(decisions),
+    )
+
+
+def load_data_go_kr_metadata_snapshot(path: Path) -> DataGoKrMetadataSnapshot:
+    """Load the offline data.go.kr semantic-evaluation metadata snapshot."""
+    raw = _load_yaml_mapping(path)
+    return DataGoKrMetadataSnapshot.model_validate(raw)
+
+
+def _gate_reason(
+    gate: GateState,
+    *,
+    failed_reason: str,
+    deferred_reason: str,
+) -> str | None:
+    if gate == "pass":
+        return None
+    if gate == "defer":
+        return deferred_reason
+    return failed_reason
+
+
+def _load_yaml_mapping(path: Path) -> dict[str, object]:
+    loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(loaded, dict):
+        raise ValueError(f"YAML fixture must be an object: {path}")
+    return cast(dict[str, object], loaded)

package/src/ummaya/tools/documents/explicit_values.py

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import re
+
+_SAVE_CLAUSE_RE = re.compile(
+    r"\s*,?\s*(?:저장|내보내|export|save)\s*(?:은|는|:|=)?.*$",
+    re.IGNORECASE,
+)
+_UPDATE_TAIL_RE = re.compile(
+    r"\s*(?:로|으로)\s*(?:(?:알아서|내용에\s*맞게)\s*)?"
+    r"(?:갱신|변경|수정|작성|입력|채우|해줘).*$",
+    re.IGNORECASE,
+)
+_NEXT_LABEL_BOUNDARY_RE = r"(?=,\s*[^,.\n]{2,80}?(?:에는|엔|에|은|는|:|=)\s*[\"'“”‘’]?)"
+_COMMAND_BOUNDARY_RE = r"(?=\s*(?:만\s*)?(?:넣|입력|작성|채우|수정|변경|갱신))"
+_VALUE_END_RE = rf"(?:{_NEXT_LABEL_BOUNDARY_RE}|{_COMMAND_BOUNDARY_RE}|[.!?。\n]|$)"
+_LABEL_PART_RE = re.compile(r"[0-9A-Za-z가-힣]+")
+_LABEL_PART_SEPARATOR_RE = r"[\s·ㆍ‧\-_]*"
+_LABEL_VALUE_SUFFIX_RE = r"(?:값|내용|value)?\s*"
+
+
+def explicit_keyed_value_for_label(label: str, instruction: str) -> str | None:
+    if not _is_meaningful_explicit_label(label):
+        return None
+    label_pattern = _explicit_label_pattern(label)
+    quoted_value = _quoted_keyed_value_for_label(label, instruction)
+    if quoted_value is not None:
+        return quoted_value
+    adjacent_blank_match = re.search(
+        rf"{label_pattern}\s*"
+        r"(?:옆|다음|우측|오른쪽)?\s*"
+        r"(?:빈\s*칸|칸|필드|항목)?\s*"
+        r"(?:에는|엔|에)\s*"
+        rf"(?P<value>.+?){_VALUE_END_RE}",
+        instruction,
+        flags=re.IGNORECASE,
+    )
+    if adjacent_blank_match is not None:
+        value = _clean_explicit_keyed_value(adjacent_blank_match.group("value"))
+        return value or None
+    field_word_match = re.search(
+        rf"{label_pattern}\s*(?:은|는|이|가)?\s*(?:필드|칸|항목)?\s*"
+        r"(?:은|는|이|가|을|를)?\s*"
+        r"(?P<value>[^.!?。\n]+?)\s*(?:로|으로)\s*"
+        r"(?:작성|입력|채우|수정|변경|갱신)",
+        instruction,
+        flags=re.IGNORECASE,
+    )
+    if field_word_match is not None:
+        value = _clean_explicit_keyed_value(field_word_match.group("value"))
+        return value or None
+    match = re.search(
+        rf"{label_pattern}\s*(?:은|는|:|=)\s*(?P<value>[^.!?。\n]+[.!?。]?)",
+        instruction,
+        flags=re.IGNORECASE,
+    )
+    if match is not None:
+        value = _clean_explicit_keyed_value(match.group("value"))
+        return value or None
+    return None
+
+
+def _quoted_keyed_value_for_label(label: str, instruction: str) -> str | None:
+    label_pattern = _explicit_label_pattern(label)
+    match = re.search(
+        rf"{label_pattern}\s*"
+        r"(?:옆|다음|우측|오른쪽)?\s*"
+        r"(?:빈\s*칸|칸|필드|항목)?\s*"
+        rf"{_LABEL_VALUE_SUFFIX_RE}"
+        r"(?:에는|엔|에|은|는|:|=)\s*"
+        r"(?P<quote>[\"'“”‘’])(?P<value>.*?)(?P=quote)",
+        instruction,
+        flags=re.IGNORECASE,
+    )
+    if match is None:
+        return None
+    value = _clean_quoted_keyed_value(match.group("value"))
+    return value or None
+
+
+def _explicit_label_pattern(label: str) -> str:
+    parts = _LABEL_PART_RE.findall(label)
+    if not parts:
+        return re.escape(label)
+    return _LABEL_PART_SEPARATOR_RE.join(re.escape(part) for part in parts)
+
+
+def _is_meaningful_explicit_label(label: str) -> bool:
+    label_key = _normalize_key(label)
+    if len(label_key) < 2:
+        return False
+    return any("a" <= char <= "z" or "가" <= char <= "힣" for char in label_key)
+
+
+def _clean_explicit_keyed_value(value: str) -> str:
+    cleaned = _SAVE_CLAUSE_RE.sub("", value).strip()
+    cleaned = _UPDATE_TAIL_RE.sub("", cleaned).strip()
+    cleaned = re.sub(r"\s*(?:을|를)$", "", cleaned).strip()
+    return cleaned.strip(" ,;:)]}）\"'“”‘’")
+
+
+def _clean_quoted_keyed_value(value: str) -> str:
+    return value.strip().strip(" ,;:)]}）\"'“”‘’")
+
+
+def _normalize_key(value: str) -> str:
+    return re.sub(r"[^0-9a-z가-힣]+", "", value.casefold())

package/src/ummaya/tools/documents/fixtures.py

@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Offline fixture manifest helpers for document harness tests."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal, cast
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+SupportedFixtureFormat = Literal["hwpx", "hwp", "docx", "pdf", "xlsx", "pptx"]
+FixtureAuthoringFlow = Literal[
+    "read_only",
+    "public_form_fill",
+    "socratic_narrative",
+    "blocked_hostile",
+]
+FixtureCoverageTag = Literal[
+    "form_blanks",
+    "self_introduction",
+    "business_plan",
+    "protected_fields",
+    "missing_evidence",
+    "approved_draft_mutation",
+    "direct_hwp_blocked",
+]
+RedistributionStatus = Literal[
+    "owned",
+    "public_domain",
+    "open_license",
+    "metadata_only",
+    "not_redistributable",
+]
+
+DEFAULT_FIXTURE_MANIFEST_PATH = (
+    Path(__file__).resolve().parents[4]
+    / "tests"
+    / "fixtures"
+    / "documents"
+    / "corpus_manifest.yaml"
+)
+
+
+class DocumentFixtureEntry(BaseModel):
+    """One offline fixture declared in the document corpus manifest."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    fixture_id: str = Field(min_length=1)
+    format: SupportedFixtureFormat
+    path: str = Field(min_length=1)
+    source: str = Field(min_length=1)
+    redistribution_status: RedistributionStatus
+    size_bytes: int | None = Field(default=None, ge=0)
+    sha256: str | None = Field(
+        default=None,
+        min_length=64,
+        max_length=64,
+        pattern=r"^[0-9a-f]{64}$",
+    )
+    authoring_flow: FixtureAuthoringFlow = "read_only"
+    expected_authoring_flow: tuple[str, ...] = ()
+    expected_fields: tuple[str, ...] = ()
+    expected_narrative_prompts: tuple[str, ...] = ()
+    expected_layout_anchors: tuple[str, ...] = ()
+    protected_field_expectations: tuple[str, ...] = ()
+    coverage_tags: tuple[FixtureCoverageTag, ...] = ()
+    negative_case: str | None = None
+
+    @field_validator(
+        "expected_authoring_flow",
+        "expected_fields",
+        "expected_narrative_prompts",
+        "expected_layout_anchors",
+        "protected_field_expectations",
+        "coverage_tags",
+    )
+    @classmethod
+    def _tuple_values_are_unique(cls, values: tuple[str, ...]) -> tuple[str, ...]:
+        if len(set(values)) != len(values):
+            raise ValueError("fixture metadata tuple values must be unique")
+        return values
+
+
+class DocumentFixtureGroups(BaseModel):
+    """Fixture groups separated by expected safety posture."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    benign: tuple[DocumentFixtureEntry, ...] = ()
+    public_forms: tuple[DocumentFixtureEntry, ...] = ()
+    narrative_authoring: tuple[DocumentFixtureEntry, ...] = ()
+    hostile: tuple[DocumentFixtureEntry, ...] = ()
+
+
+class DocumentFixtureManifest(BaseModel):
+    """Offline document fixture manifest."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    version: Literal[1]
+    manifest_id: str = Field(min_length=1)
+    created_at: str = Field(min_length=1)
+    source_policy: Literal["offline_fixtures_only"]
+    live_network_allowed: Literal[False]
+    formats: tuple[SupportedFixtureFormat, ...] = Field(min_length=1)
+    fixture_groups: DocumentFixtureGroups
+    requirements: dict[str, str]
+
+    @model_validator(mode="after")
+    def _require_known_format_groups(self) -> DocumentFixtureManifest:
+        declared = set(self.formats)
+        for entry in (
+            *self.fixture_groups.benign,
+            *self.fixture_groups.public_forms,
+            *self.fixture_groups.narrative_authoring,
+            *self.fixture_groups.hostile,
+        ):
+            if entry.format not in declared:
+                raise ValueError(
+                    f"Fixture {entry.fixture_id!r} uses undeclared format {entry.format!r}"
+                )
+        return self
+
+    @model_validator(mode="after")
+    def _require_matrix_metadata(self) -> DocumentFixtureManifest:
+        for entry in (
+            *self.fixture_groups.public_forms,
+            *self.fixture_groups.narrative_authoring,
+        ):
+            _require_common_matrix_metadata(entry)
+        for entry in self.fixture_groups.public_forms:
+            _require_public_form_matrix_metadata(entry)
+        for entry in self.fixture_groups.narrative_authoring:
+            _require_narrative_matrix_metadata(entry)
+        return self
+
+
+def load_fixture_manifest(
+    path: Path = DEFAULT_FIXTURE_MANIFEST_PATH,
+) -> DocumentFixtureManifest:
+    """Load and validate the offline document fixture manifest."""
+    raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(raw, dict):
+        raise ValueError(f"Fixture manifest must be a YAML object: {path}")
+    return DocumentFixtureManifest.model_validate(cast(dict[str, object], raw))
+
+
+def _require_common_matrix_metadata(entry: DocumentFixtureEntry) -> None:
+    if entry.size_bytes is None:
+        raise ValueError(f"Fixture {entry.fixture_id!r} must declare size_bytes")
+    if entry.sha256 is None:
+        raise ValueError(f"Fixture {entry.fixture_id!r} must declare sha256")
+    if not entry.expected_authoring_flow:
+        raise ValueError(f"Fixture {entry.fixture_id!r} must declare expected_authoring_flow")
+    if not entry.protected_field_expectations:
+        raise ValueError(f"Fixture {entry.fixture_id!r} must declare protected_field_expectations")
+    if not entry.coverage_tags:
+        raise ValueError(f"Fixture {entry.fixture_id!r} must declare coverage_tags")
+
+
+def _require_public_form_matrix_metadata(entry: DocumentFixtureEntry) -> None:
+    if entry.authoring_flow != "public_form_fill":
+        raise ValueError(f"Public form fixture {entry.fixture_id!r} has wrong flow")
+    if not entry.expected_fields:
+        raise ValueError(f"Public form fixture {entry.fixture_id!r} needs fields")
+
+
+def _require_narrative_matrix_metadata(entry: DocumentFixtureEntry) -> None:
+    if entry.authoring_flow != "socratic_narrative":
+        raise ValueError(f"Narrative fixture {entry.fixture_id!r} has wrong flow")
+    if not entry.expected_narrative_prompts:
+        raise ValueError(f"Narrative fixture {entry.fixture_id!r} needs narrative prompts")