millforge 0.1.0__py3-none-any.whl

millforge/compiler/__init__.py

@@ -0,0 +1,231 @@
+"""Compiler front-end contracts for Millforge harness sources."""
+
+from millforge.compiler.diagnostics import (
+    DIAGNOSTIC_REGISTRY,
+    DIAGNOSTIC_TRIGGER_MEANINGS,
+    CompilerDiagnostic,
+    CompilerPhase,
+    DiagnosticField,
+    DiagnosticSeverity,
+    SourceLocation,
+    SourceReference,
+    detect_secret_candidate,
+    sort_diagnostics,
+)
+from millforge.compiler.catalogs import (
+    CatalogLookupClassification,
+    CatalogMetadataError,
+    CatalogSnapshotMetadata,
+    ModelProfileCatalogLookup,
+    ModelProfileCatalogSnapshot,
+    RawToolDescriptor,
+    ToolCatalogEntry,
+    ToolCatalogLookup,
+    ToolCatalogSnapshot,
+    admit_model_profile,
+    capture_catalog_snapshot_metadata,
+)
+from millforge.compiler.artifact_validation import (
+    ArtifactProducerEvidence,
+    ArtifactValidationResult,
+    validate_artifacts,
+)
+from millforge.compiler.capabilities import (
+    CapabilityValidationResult,
+    validate_capability_grants,
+)
+from millforge.compiler.canonicalization import (
+    canonical_semantic_bytes,
+    canonical_semantic_payload,
+    source_sha256,
+)
+from millforge.compiler.graph import (
+    GraphValidationResult,
+    ResolvedNodeDescriptor,
+    validate_harness_graph,
+)
+from millforge.compiler.lowering import (
+    COMPILER_BUILD_ID,
+    COMPILER_NAME,
+    CompiledPlanValidationError,
+    LoweringInvariantError,
+    SourceSemanticHashError,
+    compiler_identity,
+    lower_resolved_harness,
+)
+from millforge.compiler.output import (
+    compiled_plan_output_path,
+    diagnostics_output_path,
+    persist_compile_outputs,
+    request_identity_sha256,
+)
+from millforge.compiler.parsing import (
+    HarnessSourceParser,
+    ParsedHarnessSource,
+    SourceDocument,
+)
+from millforge.compiler.requests import (
+    CompileInvocation,
+    CompileStatus,
+    DefaultHarnessCompileRequestAdmission,
+    DiagnosticReportState,
+    HarnessCompileRequest,
+    HarnessCompileRequestAdmission,
+    HarnessCompileResult,
+    HarnessRequestAdmissionResult,
+    PlanCommitCertainty,
+)
+from millforge.compiler.service import (
+    HarnessCompiler,
+    InMemoryHarnessCompileError,
+    compile,
+    compile_harness_source_in_memory,
+    compile_raw,
+)
+from millforge.compiler.semantic import (
+    ResolvedHarness,
+    ResolvedToolBinding,
+    ResolvedToolBindingRef,
+    SemanticCompileResult,
+    compile_semantic,
+    compile_semantic_from_admission,
+)
+from millforge.compiler.schema_validation import (
+    SCHEMA_ALLOWED_KEYWORDS,
+    SCHEMA_ANNOTATION_KEYWORDS,
+    SCHEMA_STRUCTURAL_KEYWORDS,
+    SchemaSubsetError,
+    normalize_json_schema,
+    normalized_schema_bytes,
+    property_schema_compatibility_bytes,
+    validate_json_schema_subset,
+)
+from millforge.compiler.source import (
+    ArgumentMatchSource,
+    ArtifactPolicySource,
+    BudgetSource,
+    ContextPolicySource,
+    HarnessGraphSource,
+    HarnessNodeSource,
+    HarnessSource,
+    PrerequisiteSource,
+    PromptSource,
+    StageScopeSource,
+    TerminalArtifactPolicySource,
+)
+from millforge.compiler.validators import (
+    ToolReference,
+    parse_tool_reference,
+    validate_argument_name,
+    validate_artifact_id,
+    validate_canonical_tool_id,
+    validate_harness_id,
+    validate_node_id,
+    validate_policy_id,
+    validate_profile_id,
+    validate_request_id,
+    validate_stage_kind_id,
+    validate_terminal_result,
+    validate_tool_reference,
+)
+
+__all__ = [
+    "ArgumentMatchSource",
+    "ArtifactProducerEvidence",
+    "ArtifactPolicySource",
+    "ArtifactValidationResult",
+    "BudgetSource",
+    "CapabilityValidationResult",
+    "CatalogLookupClassification",
+    "CatalogMetadataError",
+    "CatalogSnapshotMetadata",
+    "canonical_semantic_bytes",
+    "canonical_semantic_payload",
+    "CompileInvocation",
+    "CompileStatus",
+    "CompilerDiagnostic",
+    "CompilerPhase",
+    "ContextPolicySource",
+    "DIAGNOSTIC_REGISTRY",
+    "DIAGNOSTIC_TRIGGER_MEANINGS",
+    "DefaultHarnessCompileRequestAdmission",
+    "DiagnosticReportState",
+    "DiagnosticField",
+    "DiagnosticSeverity",
+    "HarnessCompileRequest",
+    "HarnessCompileRequestAdmission",
+    "HarnessCompileResult",
+    "HarnessCompiler",
+    "InMemoryHarnessCompileError",
+    "HarnessGraphSource",
+    "HarnessNodeSource",
+    "HarnessRequestAdmissionResult",
+    "HarnessSource",
+    "HarnessSourceParser",
+    "GraphValidationResult",
+    "LoweringInvariantError",
+    "COMPILER_BUILD_ID",
+    "COMPILER_NAME",
+    "CompiledPlanValidationError",
+    "ModelProfileCatalogLookup",
+    "ModelProfileCatalogSnapshot",
+    "ParsedHarnessSource",
+    "PlanCommitCertainty",
+    "PrerequisiteSource",
+    "PromptSource",
+    "SourceDocument",
+    "SourceLocation",
+    "SourceReference",
+    "StageScopeSource",
+    "TerminalArtifactPolicySource",
+    "RawToolDescriptor",
+    "ResolvedHarness",
+    "ResolvedNodeDescriptor",
+    "ResolvedToolBinding",
+    "ResolvedToolBindingRef",
+    "SCHEMA_ALLOWED_KEYWORDS",
+    "SCHEMA_ANNOTATION_KEYWORDS",
+    "SCHEMA_STRUCTURAL_KEYWORDS",
+    "SchemaSubsetError",
+    "SemanticCompileResult",
+    "SourceSemanticHashError",
+    "source_sha256",
+    "ToolCatalogEntry",
+    "ToolCatalogLookup",
+    "ToolCatalogSnapshot",
+    "ToolReference",
+    "admit_model_profile",
+    "capture_catalog_snapshot_metadata",
+    "compiler_identity",
+    "compiled_plan_output_path",
+    "compile",
+    "compile_harness_source_in_memory",
+    "compile_raw",
+    "compile_semantic",
+    "compile_semantic_from_admission",
+    "diagnostics_output_path",
+    "detect_secret_candidate",
+    "normalize_json_schema",
+    "normalized_schema_bytes",
+    "lower_resolved_harness",
+    "persist_compile_outputs",
+    "parse_tool_reference",
+    "property_schema_compatibility_bytes",
+    "request_identity_sha256",
+    "sort_diagnostics",
+    "validate_artifacts",
+    "validate_argument_name",
+    "validate_artifact_id",
+    "validate_canonical_tool_id",
+    "validate_harness_id",
+    "validate_harness_graph",
+    "validate_capability_grants",
+    "validate_node_id",
+    "validate_policy_id",
+    "validate_profile_id",
+    "validate_request_id",
+    "validate_stage_kind_id",
+    "validate_terminal_result",
+    "validate_tool_reference",
+    "validate_json_schema_subset",
+]

millforge/compiler/artifact_validation.py

@@ -0,0 +1,257 @@
+"""Artifact satisfiability validation for semantic compiler resolution."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Mapping
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+
+from millforge.compiler.catalogs import ToolCatalogEntry
+from millforge.compiler.diagnostics import (
+    CompilerDiagnostic,
+    CompilerPhase,
+    DiagnosticField,
+    DiagnosticSeverity,
+    sort_diagnostics,
+)
+from millforge.compiler.graph import GraphValidationResult, ResolvedNodeDescriptor
+from millforge.compiler.source import HarnessNodeSource, HarnessSource
+
+
+class ArtifactProducerEvidence(BaseModel):
+    """Producer evidence for one artifact ID."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    artifact_id: StrictStr
+    all_producer_node_ids: tuple[StrictStr, ...] = Field(default_factory=tuple)
+    terminal_gated_producer_node_ids: tuple[StrictStr, ...] = Field(
+        default_factory=tuple
+    )
+
+
+class ArtifactValidationResult(BaseModel):
+    """Immutable artifact validation result."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    diagnostics: tuple[CompilerDiagnostic, ...] = Field(default_factory=tuple)
+    producer_evidence: tuple[ArtifactProducerEvidence, ...] = Field(
+        default_factory=tuple
+    )
+
+    @property
+    def ok(self) -> bool:
+        return not self.diagnostics
+
+
+def validate_artifacts(
+    source: HarnessSource,
+    resolved_nodes: Mapping[str, ToolCatalogEntry | ResolvedNodeDescriptor],
+    graph_result: GraphValidationResult,
+) -> ArtifactValidationResult:
+    """Validate source artifact declarations against resolved producer evidence."""
+    nodes_by_id = {node.node_id: node for node in source.graph.nodes}
+    descriptors = _resolved_entries(resolved_nodes)
+    declared = set(source.artifacts.declared_artifact_ids)
+    diagnostics: list[CompilerDiagnostic] = []
+    valid_producers: dict[str, list[str]] = defaultdict(list)
+
+    for artifact_id in _duplicates(source.artifacts.declared_artifact_ids):
+        diagnostics.append(
+            _diagnostic(
+                "MF-A006",
+                f"Declared artifact {artifact_id!r} is duplicated.",
+                artifact_id=artifact_id,
+            )
+        )
+
+    for node in sorted(source.graph.nodes, key=lambda item: item.node_id):
+        descriptor = descriptors.get(node.node_id)
+        if descriptor is None:
+            continue
+        descriptor_artifacts = set(descriptor.produced_artifact_ids)
+        for artifact_id in _duplicates(node.produces):
+            diagnostics.append(
+                _diagnostic(
+                    "MF-A006",
+                    f"Node {node.node_id!r} duplicates produced artifact {artifact_id!r}.",
+                    node_id=node.node_id,
+                    artifact_id=artifact_id,
+                )
+            )
+        for artifact_id in sorted(set(node.produces)):
+            if artifact_id not in declared:
+                diagnostics.append(
+                    _diagnostic(
+                        "MF-A001",
+                        f"Node {node.node_id!r} produces undeclared artifact {artifact_id!r}.",
+                        node_id=node.node_id,
+                        artifact_id=artifact_id,
+                    )
+                )
+            if artifact_id not in descriptor_artifacts:
+                diagnostics.append(
+                    _diagnostic(
+                        "MF-A002",
+                        f"Node {node.node_id!r} cannot produce artifact {artifact_id!r}.",
+                        node_id=node.node_id,
+                        artifact_id=artifact_id,
+                    )
+                )
+            if artifact_id in declared and artifact_id in descriptor_artifacts:
+                valid_producers[artifact_id].append(node.node_id)
+
+    terminal_results = set(graph_result.terminal_result_map.values())
+    terminal_nodes_by_result = {
+        result: node_id for node_id, result in graph_result.terminal_result_map.items()
+    }
+    for requirement in source.artifacts.required_by_terminal:
+        for artifact_id in _duplicates(requirement.artifact_ids):
+            diagnostics.append(
+                _diagnostic(
+                    "MF-A006",
+                    f"Terminal artifact policy {requirement.terminal_result!r} duplicates artifact {artifact_id!r}.",
+                    artifact_id=artifact_id,
+                    terminal_result=requirement.terminal_result,
+                )
+            )
+        if requirement.terminal_result not in terminal_results:
+            diagnostics.append(
+                _diagnostic(
+                    "MF-A003",
+                    f"Artifact policy references unknown terminal result {requirement.terminal_result!r}.",
+                    terminal_result=requirement.terminal_result,
+                )
+            )
+            continue
+        for artifact_id in sorted(set(requirement.artifact_ids)):
+            if artifact_id not in declared:
+                diagnostics.append(
+                    _diagnostic(
+                        "MF-A007",
+                        f"Terminal-required artifact {artifact_id!r} is not declared.",
+                        artifact_id=artifact_id,
+                        terminal_result=requirement.terminal_result,
+                    )
+                )
+
+    producer_evidence: list[ArtifactProducerEvidence] = []
+    terminal_gated: dict[str, set[str]] = defaultdict(set)
+    if not graph_result.diagnostics:
+        ancestors_by_terminal = {
+            terminal_node_id: _ancestors(terminal_node_id, nodes_by_id)
+            for terminal_node_id in graph_result.terminal_node_ids
+        }
+        globally_required = set(graph_result.required_node_ids)
+        for requirement in source.artifacts.required_by_terminal:
+            terminal_node_id = terminal_nodes_by_result.get(requirement.terminal_result)
+            if terminal_node_id is None:
+                continue
+            gated_nodes = ancestors_by_terminal[terminal_node_id] | globally_required
+            for artifact_id in sorted(set(requirement.artifact_ids)):
+                if artifact_id not in declared:
+                    continue
+                producers = set(valid_producers.get(artifact_id, ()))
+                gated_producers = producers & gated_nodes
+                terminal_gated[artifact_id].update(gated_producers)
+                if not producers:
+                    diagnostics.append(
+                        _diagnostic(
+                            "MF-A004",
+                            f"Terminal-required artifact {artifact_id!r} has no valid producer.",
+                            artifact_id=artifact_id,
+                            terminal_result=requirement.terminal_result,
+                        )
+                    )
+                elif not gated_producers:
+                    diagnostics.append(
+                        _diagnostic(
+                            "MF-A005",
+                            f"Terminal-required artifact {artifact_id!r} has no terminal-gated producer.",
+                            artifact_id=artifact_id,
+                            terminal_result=requirement.terminal_result,
+                        )
+                    )
+
+    for artifact_id in sorted(declared):
+        producer_evidence.append(
+            ArtifactProducerEvidence(
+                artifact_id=artifact_id,
+                all_producer_node_ids=tuple(
+                    sorted(valid_producers.get(artifact_id, ()))
+                ),
+                terminal_gated_producer_node_ids=tuple(
+                    sorted(terminal_gated.get(artifact_id, ()))
+                ),
+            )
+        )
+
+    return ArtifactValidationResult(
+        diagnostics=sort_diagnostics(diagnostics),
+        producer_evidence=tuple(producer_evidence),
+    )
+
+
+def _resolved_entries(
+    resolved_nodes: Mapping[str, ToolCatalogEntry | ResolvedNodeDescriptor],
+) -> dict[str, ToolCatalogEntry]:
+    entries: dict[str, ToolCatalogEntry] = {}
+    for node_id, value in resolved_nodes.items():
+        entries[node_id] = (
+            value.descriptor if isinstance(value, ResolvedNodeDescriptor) else value
+        )
+    return entries
+
+
+def _ancestors(
+    terminal_node_id: str,
+    nodes_by_id: Mapping[str, HarnessNodeSource],
+) -> set[str]:
+    reverse = {
+        node.node_id: tuple(prereq.node_id for prereq in node.prerequisites)
+        for node in nodes_by_id.values()
+    }
+    ancestors: set[str] = set()
+    stack = list(reverse[terminal_node_id])
+    while stack:
+        node_id = stack.pop()
+        if node_id in ancestors or node_id not in reverse:
+            continue
+        ancestors.add(node_id)
+        stack.extend(reverse[node_id])
+    return ancestors
+
+
+def _duplicates(values: tuple[str, ...]) -> tuple[str, ...]:
+    seen: set[str] = set()
+    duplicates: set[str] = set()
+    for value in values:
+        if value in seen:
+            duplicates.add(value)
+        seen.add(value)
+    return tuple(sorted(duplicates))
+
+
+def _diagnostic(
+    code: str,
+    message: str,
+    *,
+    node_id: str | None = None,
+    artifact_id: str | None = None,
+    terminal_result: str | None = None,
+) -> CompilerDiagnostic:
+    fields = []
+    if artifact_id is not None:
+        fields.append(DiagnosticField(key="artifact_id", value=artifact_id))
+    if terminal_result is not None:
+        fields.append(DiagnosticField(key="terminal_result", value=terminal_result))
+    return CompilerDiagnostic(
+        code=code,
+        phase=CompilerPhase.ARTIFACT,
+        severity=DiagnosticSeverity.ERROR,
+        message=message,
+        node_id=node_id,
+        fields=tuple(fields),
+    )

millforge/compiler/canonicalization.py

@@ -0,0 +1,169 @@
+"""Canonical semantic payload construction for deterministic compiler hashing."""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+from collections.abc import Mapping
+from typing import Any, TypeAlias
+
+from millforge import canonical_json_serialize
+from millforge.compiler.semantic import ResolvedHarness
+
+JsonScalar: TypeAlias = str | int | float | bool | None
+JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"]
+JsonObject: TypeAlias = dict[str, JsonValue]
+
+
+def canonical_semantic_payload(resolved: ResolvedHarness) -> JsonObject:
+    """Return the validated source semantics used for ``source_sha256``.
+
+    The payload is built from fresh JSON-compatible containers and excludes
+    request identity, parser metadata, source locations, output paths, catalog
+    snapshot identity, and catalog-owned objects.
+    """
+    source = resolved.source
+    return _fresh_json_object(
+        {
+            "schema_version": source.schema_version,
+            "kind": source.kind,
+            "harness_id": source.harness_id,
+            "harness_version": source.harness_version,
+            "stage_kind_ids": sorted(source.stage_scope.stage_kind_ids),
+            "model_profile": _fresh_json_value(
+                resolved.model_profile.model_dump(mode="json")
+            ),
+            "prompt": {
+                "policy_id": source.prompt.policy_id,
+                "system_instructions": source.prompt.system_instructions,
+                "include_request_context": source.prompt.include_request_context,
+            },
+            "budgets": {
+                "max_iterations": source.budgets.max_iterations,
+                "max_validation_retries": source.budgets.max_validation_retries,
+                "max_tool_errors": source.budgets.max_tool_errors,
+                "max_prerequisite_violations": (
+                    source.budgets.max_prerequisite_violations
+                ),
+                "max_premature_terminal_attempts": (
+                    source.budgets.max_premature_terminal_attempts
+                ),
+            },
+            "context": {
+                "strategy_id": source.context.strategy_id,
+                "budget_tokens": source.context.budget_tokens,
+                "keep_recent_iterations": source.context.keep_recent_iterations,
+                "phase_thresholds": list(source.context.phase_thresholds),
+            },
+            "nodes": [_node_payload(node) for node in resolved.resolved_nodes],
+            "required_node_ids": sorted(resolved.required_node_ids),
+            "required_capability_ids": sorted(resolved.required_capability_ids),
+            "terminal_result_map": dict(sorted(resolved.terminal_result_map.items())),
+            "artifacts": {
+                "declared_artifact_ids": sorted(source.artifacts.declared_artifact_ids),
+                "required_by_terminal": [
+                    {
+                        "terminal_result": item.terminal_result,
+                        "artifact_ids": sorted(item.artifact_ids),
+                    }
+                    for item in sorted(
+                        source.artifacts.required_by_terminal,
+                        key=lambda item: item.terminal_result,
+                    )
+                ],
+                "producer_evidence": [
+                    {
+                        "artifact_id": item.artifact_id,
+                        "all_producer_node_ids": sorted(item.all_producer_node_ids),
+                        "terminal_gated_producer_node_ids": sorted(
+                            item.terminal_gated_producer_node_ids
+                        ),
+                    }
+                    for item in sorted(
+                        resolved.artifact_evidence,
+                        key=lambda item: item.artifact_id,
+                    )
+                ],
+            },
+        }
+    )
+
+
+def canonical_semantic_bytes(resolved: ResolvedHarness) -> bytes:
+    """Return UTF-8 canonical JSON bytes for a resolved semantic payload."""
+    return canonical_json_serialize(canonical_semantic_payload(resolved)).encode(
+        "utf-8"
+    )
+
+
+def source_sha256(resolved: ResolvedHarness) -> str:
+    """Return SHA-256 over the canonical validated semantic payload."""
+    return hashlib.sha256(canonical_semantic_bytes(resolved)).hexdigest()
+
+
+def _node_payload(node: Any) -> JsonObject:
+    source = node.source
+    descriptor = node.descriptor
+    return _fresh_json_object(
+        {
+            "node_id": node.node_id,
+            "required": source.required,
+            "terminal_result": source.terminal_result,
+            "prerequisites": [
+                {
+                    "node_id": prerequisite.node_id,
+                    "argument_matches": [
+                        {
+                            "prior_argument": match.prior_argument,
+                            "current_argument": match.current_argument,
+                        }
+                        for match in sorted(
+                            prerequisite.argument_matches,
+                            key=lambda item: (
+                                item.prior_argument,
+                                item.current_argument,
+                            ),
+                        )
+                    ],
+                }
+                for prerequisite in sorted(
+                    source.prerequisites, key=lambda item: item.node_id
+                )
+            ],
+            "produces": sorted(source.produces),
+            "binding": {
+                "tool_id": node.binding.tool_id,
+                "tool_version": node.binding.tool_version,
+                "descriptor_sha256": node.binding.descriptor_sha256,
+                "implementation_id": node.binding.implementation_id,
+            },
+            "descriptor": {
+                "model_tool_name": descriptor.model_tool_name,
+                "description": descriptor.description,
+                "input_schema": _fresh_json_value(descriptor.input_schema),
+                "side_effect_class": descriptor.side_effect_class.value,
+                "idempotency": descriptor.idempotency.value,
+                "required_capabilities": sorted(descriptor.required_capabilities),
+                "produced_artifact_ids": sorted(descriptor.produced_artifact_ids),
+            },
+        }
+    )
+
+
+def _fresh_json_object(value: Mapping[str, Any]) -> JsonObject:
+    converted = _fresh_json_value(value)
+    if not isinstance(converted, dict):
+        raise TypeError("expected JSON object")
+    return converted
+
+
+def _fresh_json_value(value: Any) -> JsonValue:
+    if value is None or isinstance(value, str | int | bool):
+        return value
+    if isinstance(value, float):
+        return value
+    if isinstance(value, Mapping):
+        return {str(key): _fresh_json_value(item) for key, item in value.items()}
+    if isinstance(value, tuple | list):
+        return [_fresh_json_value(item) for item in value]
+    return copy.deepcopy(value)