millforge 0.1.0__py3-none-any.whl

millforge/base/composition.py

@@ -0,0 +1,402 @@
+"""Composition boundary for the unrestricted Millforge base preset."""
+
+from __future__ import annotations
+
+import datetime
+import hashlib
+import platform
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    StrictBool,
+    StrictInt,
+    StrictStr,
+    field_validator,
+)
+
+from millforge.compiled_plan import CompiledHarnessPlan, CompiledModelProfile
+from millforge.compiler.catalogs import ModelProfileCatalogLookup, ToolCatalogSnapshot
+from millforge.compiler.service import compile_harness_source_in_memory
+from millforge.compiler.source import HarnessSource
+from millforge.contracts import CapabilityEnvelope, CapabilityGrant
+from millforge.model_backend import (
+    CapabilitySupport,
+    ResolvedModelProfile,
+    UnsupportedModelCapabilityError,
+)
+from millforge.protocols import CancellationResolver
+from millforge.tools.execution import CompiledToolBindingExecutor
+from millforge.tools.pi_compat.process import (
+    PiCompatShellConfig,
+    resolve_pi_compat_shell,
+)
+from millforge.tools.pi_compat_catalog import (
+    PI_COMPAT_TOOL_DESCRIPTORS,
+    _create_pi_compat_tool_snapshot_for_terminal_results,
+)
+from millforge.tools.pi_compat_runtime import (
+    _create_pi_compat_tool_executor_for_terminal_results,
+)
+
+from .context import (
+    MillforgeBaseContextSnapshot,
+    _load_millforge_base_context_resolved,
+)
+from .harness import (
+    _CONFIG_ID,
+    _HARNESS_ID,
+    _STAGE_KIND,
+    _TOOL_PACK_ID,
+    DEFAULT_BASE_TERMINAL_RESULTS,
+    _millforge_base_harness_source_for_terminal_results,
+    canonicalize_base_terminal_results,
+)
+from .options import MillforgeBaseOptions
+from .platform import _require_supported_platform
+from .prompt import (
+    MillforgeBasePromptSnapshot,
+    _build_millforge_base_system_prompt_resolved,
+)
+
+__all__ = [
+    "MillforgeBaseComponents",
+    "MillforgeBaseMetadata",
+    "create_millforge_base_components",
+]
+
+_UPSTREAM_PACKAGE = "@earendil-works/pi-coding-agent"
+_UPSTREAM_VERSION = "0.79.6"
+_COMPATIBILITY_CLAIM = (
+    "A Python behavioral port of Pi 0.79.6's complete built-in coding tool pack, "
+    "adapted to Millforge's compiler and runtime contracts."
+)
+_SECURITY_WARNING = (
+    "millforge-base runs with the permissions of the Millforge process. It can read, "
+    "write, delete, execute commands, access the network, and access credentials "
+    "available to that process. Use only in trusted environments."
+)
+_ENABLED_ALIASES = (
+    "read",
+    "bash",
+    "edit",
+    "write",
+    "grep",
+    "find",
+    "ls",
+    "submit",
+    "block",
+    "reject",
+)
+_SHA256_RE = re.compile(r"[0-9a-f]{64}")
+
+
+class MillforgeBaseMetadata(BaseModel):
+    """Sanitized, deterministic metadata for one composed base preset."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: Literal[1]
+    config_id: StrictStr
+    harness_id: StrictStr
+    tool_pack_id: StrictStr
+    upstream_package: StrictStr
+    upstream_version: StrictStr
+    compatibility_claim: StrictStr
+    security_warning: StrictStr
+    unrestricted: Literal[True]
+    enabled_aliases: tuple[StrictStr, ...]
+    model_profile_id: StrictStr
+    provider_id: StrictStr
+    model_id: StrictStr
+    transport_id: StrictStr
+    os_name: StrictStr
+    shell_name: StrictStr
+    cwd_sha256: StrictStr
+    descriptor_snapshot_sha256: StrictStr
+    compiled_sha256: StrictStr
+    effective_prompt_sha256: StrictStr
+    context_sha256: StrictStr
+    context_file_count: StrictInt = Field(ge=0)
+    context_truncated: StrictBool
+    prompt_truncated: StrictBool
+
+    @field_validator(
+        "config_id",
+        "harness_id",
+        "tool_pack_id",
+        "upstream_package",
+        "upstream_version",
+        "compatibility_claim",
+        "security_warning",
+        "model_profile_id",
+        "provider_id",
+        "model_id",
+        "transport_id",
+        "os_name",
+        "shell_name",
+    )
+    @classmethod
+    def _strings_are_nonblank(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("metadata strings must be nonblank")
+        return value
+
+    @field_validator("enabled_aliases")
+    @classmethod
+    def _aliases_are_nonblank(cls, value: tuple[str, ...]) -> tuple[str, ...]:
+        if not value or any(not alias.strip() for alias in value):
+            raise ValueError("enabled_aliases must contain only nonblank values")
+        return value
+
+    @field_validator(
+        "cwd_sha256",
+        "descriptor_snapshot_sha256",
+        "compiled_sha256",
+        "effective_prompt_sha256",
+        "context_sha256",
+    )
+    @classmethod
+    def _hashes_are_lowercase_sha256(cls, value: str) -> str:
+        if not _SHA256_RE.fullmatch(value):
+            raise ValueError("metadata hashes must be lowercase SHA-256 values")
+        return value
+
+
+@dataclass(frozen=True)
+class MillforgeBaseComponents:
+    """Fully composed transient components for one base-preset invocation."""
+
+    legal_terminal_results: tuple[str, ...]
+    options: MillforgeBaseOptions
+    context: MillforgeBaseContextSnapshot
+    prompt: MillforgeBasePromptSnapshot
+    harness_source: HarnessSource
+    compiled_plan: CompiledHarnessPlan
+    model_profile: ResolvedModelProfile
+    tool_snapshot: ToolCatalogSnapshot
+    tool_executor: CompiledToolBindingExecutor
+    capability_envelope: CapabilityEnvelope
+    metadata: MillforgeBaseMetadata
+
+
+@dataclass(frozen=True)
+class _SingleModelProfileCatalog:
+    """One admitted logical model profile for a single in-memory compilation."""
+
+    profile: CompiledModelProfile
+    snapshot_id: str
+    snapshot_sha256: str
+
+    def resolve_exact(self, profile_id: str) -> ModelProfileCatalogLookup:
+        if profile_id == self.profile.profile_id:
+            return ModelProfileCatalogLookup.found(self.profile)
+        return ModelProfileCatalogLookup.missing(
+            error_code="profile.missing",
+            evidence={"profile_id": profile_id},
+        )
+
+
+def _resolved_absolute_path(path: Path, field_name: str) -> Path:
+    if not path.is_absolute():
+        raise ValueError(f"{field_name} must be absolute")
+    return path.resolve()
+
+
+def _sha256(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def _capability_envelope() -> CapabilityEnvelope:
+    required_capabilities = sorted(
+        {
+            capability_id
+            for descriptor in PI_COMPAT_TOOL_DESCRIPTORS
+            for capability_id in descriptor.required_capabilities
+        }
+    )
+    return CapabilityEnvelope(
+        grants=tuple(
+            CapabilityGrant(capability_id=capability_id)
+            for capability_id in required_capabilities
+        )
+    )
+
+
+def _model_profile_catalog(profile_id: str) -> _SingleModelProfileCatalog:
+    snapshot_id = _sha256(f"millforge-base.model-profile:{profile_id}")
+    return _SingleModelProfileCatalog(
+        profile=CompiledModelProfile(profile_id=profile_id),
+        snapshot_id=snapshot_id,
+        snapshot_sha256=_sha256(f"millforge-base.model-profile.snapshot:{profile_id}"),
+    )
+
+
+def _metadata(
+    *,
+    model_profile: ResolvedModelProfile,
+    cwd: Path,
+    shell_config: PiCompatShellConfig,
+    context: MillforgeBaseContextSnapshot,
+    prompt: MillforgeBasePromptSnapshot,
+    tool_snapshot: ToolCatalogSnapshot,
+    compiled_plan: CompiledHarnessPlan,
+) -> MillforgeBaseMetadata:
+    return MillforgeBaseMetadata(
+        schema_version=1,
+        config_id=_CONFIG_ID,
+        harness_id=_HARNESS_ID,
+        tool_pack_id=_TOOL_PACK_ID,
+        upstream_package=_UPSTREAM_PACKAGE,
+        upstream_version=_UPSTREAM_VERSION,
+        compatibility_claim=_COMPATIBILITY_CLAIM,
+        security_warning=_SECURITY_WARNING,
+        unrestricted=True,
+        enabled_aliases=_ENABLED_ALIASES,
+        model_profile_id=model_profile.profile_id,
+        provider_id=model_profile.provider_id,
+        model_id=model_profile.model_id,
+        transport_id=model_profile.transport_id,
+        os_name=platform.system().lower(),
+        shell_name=Path(shell_config.executable).name,
+        cwd_sha256=_sha256(cwd.as_posix()),
+        descriptor_snapshot_sha256=tool_snapshot.snapshot_sha256,
+        compiled_sha256=compiled_plan.compiled_sha256,
+        effective_prompt_sha256=prompt.effective_prompt_sha256,
+        context_sha256=context.context_sha256,
+        context_file_count=len(context.files),
+        context_truncated=context.truncated,
+        prompt_truncated=prompt.truncated,
+    )
+
+
+def create_millforge_base_components(
+    *,
+    legal_terminal_results: tuple[str, ...] = DEFAULT_BASE_TERMINAL_RESULTS,
+    model_profile: ResolvedModelProfile,
+    cwd: Path,
+    cancellation_resolver: CancellationResolver,
+    options: MillforgeBaseOptions | None = None,
+    prompt_date: datetime.date | None = None,
+    home_directory: Path | None = None,
+) -> MillforgeBaseComponents:
+    """Compose the unrestricted preset without making a provider request."""
+
+    return _create_millforge_base_components(
+        legal_terminal_results=canonicalize_base_terminal_results(
+            legal_terminal_results
+        ),
+        model_profile=model_profile,
+        cwd=cwd,
+        cancellation_resolver=cancellation_resolver,
+        options=options,
+        prompt_date=prompt_date,
+        home_directory=home_directory,
+    )
+
+
+def _create_millforge_base_components(
+    *,
+    legal_terminal_results: tuple[str, ...],
+    model_profile: ResolvedModelProfile,
+    cwd: Path,
+    cancellation_resolver: CancellationResolver,
+    options: MillforgeBaseOptions | None = None,
+    prompt_date: datetime.date | None = None,
+    home_directory: Path | None = None,
+) -> MillforgeBaseComponents:
+    """Compose a base preset from already-canonical terminal configuration."""
+
+    _require_supported_platform()
+
+    unsupported_model_capabilities = tuple(
+        capability
+        for capability in ("tool_calls", "system_messages", "tool_result_messages")
+        if model_profile.capabilities.state_for(capability)
+        is not CapabilitySupport.SUPPORTED
+    )
+    if unsupported_model_capabilities:
+        raise UnsupportedModelCapabilityError(
+            "millforge-base requires supported model capabilities: "
+            + ", ".join(unsupported_model_capabilities)
+        )
+
+    resolved_cwd = _resolved_absolute_path(cwd, "cwd")
+    resolved_home = _resolved_absolute_path(
+        Path.home() if home_directory is None else home_directory,
+        "home_directory",
+    )
+    effective_options = options or MillforgeBaseOptions()
+    effective_prompt_date = prompt_date or datetime.date.today()
+    context = _load_millforge_base_context_resolved(
+        cwd=resolved_cwd,
+        home_directory=resolved_home,
+        enabled=effective_options.load_context_files,
+    )
+    prompt = _build_millforge_base_system_prompt_resolved(
+        options=effective_options,
+        context=context,
+        cwd=resolved_cwd,
+        home_directory=resolved_home,
+        prompt_date=effective_prompt_date,
+    )
+    shell_config = resolve_pi_compat_shell()
+    source = _millforge_base_harness_source_for_terminal_results(
+        model_profile_id=model_profile.profile_id,
+        system_instructions=prompt.system_instructions,
+        legal_terminal_results=legal_terminal_results,
+    )
+    tool_snapshot = _create_pi_compat_tool_snapshot_for_terminal_results(
+        legal_terminal_results
+    )
+    capability_envelope = _capability_envelope()
+    compiled_plan = compile_harness_source_in_memory(
+        request_id="millforge-base.v1",
+        source=source,
+        stage_kind_id=_STAGE_KIND,
+        legal_terminal_results=legal_terminal_results,
+        capability_envelope=capability_envelope,
+        tool_catalog=tool_snapshot,
+        model_profile_catalog=_model_profile_catalog(model_profile.profile_id),
+    )
+    if tuple(
+        grant.capability_id for grant in capability_envelope.grants
+    ) != compiled_plan.required_capabilities or any(
+        grant.constraints is not None for grant in capability_envelope.grants
+    ):
+        raise RuntimeError("millforge-base capability envelope does not match its plan")
+
+    tool_executor = _create_pi_compat_tool_executor_for_terminal_results(
+        compiled_plan,
+        cwd=resolved_cwd,
+        cancellation_resolver=cancellation_resolver,
+        shell_config=shell_config,
+        legal_terminal_results=legal_terminal_results,
+    )
+    metadata = _metadata(
+        model_profile=model_profile,
+        cwd=resolved_cwd,
+        shell_config=shell_config,
+        context=context,
+        prompt=prompt,
+        tool_snapshot=tool_snapshot,
+        compiled_plan=compiled_plan,
+    )
+    return MillforgeBaseComponents(
+        legal_terminal_results=legal_terminal_results,
+        options=effective_options,
+        context=context,
+        prompt=prompt,
+        harness_source=source,
+        compiled_plan=compiled_plan,
+        model_profile=model_profile,
+        tool_snapshot=tool_snapshot,
+        tool_executor=tool_executor,
+        capability_envelope=capability_envelope,
+        metadata=metadata,
+    )

millforge/base/context.py

@@ -0,0 +1,285 @@
+"""Pi-derived project-context discovery for ``millforge-base``."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    StrictBool,
+    StrictInt,
+    StrictStr,
+    field_validator,
+)
+
+__all__ = [
+    "MillforgeBaseContextFile",
+    "MillforgeBaseContextSnapshot",
+    "load_millforge_base_context",
+]
+
+_CONTEXT_BYTE_LIMIT = 49_152
+_DIAGNOSTIC_BYTE_LIMIT = 2_048
+_CONTEXT_CANDIDATES = ("AGENTS.md", "AGENTS.MD", "CLAUDE.md", "CLAUDE.MD")
+_SHA256_RE = re.compile(r"[0-9a-f]{64}")
+
+
+def _truncate_to_utf8_bytes(value: str, limit: int) -> str:
+    """Keep a valid UTF-8 prefix within ``limit`` bytes."""
+
+    encoded = value.encode("utf-8")
+    if len(encoded) <= limit:
+        return value
+    return encoded[:limit].decode("utf-8", errors="ignore")
+
+
+def _bounded_diagnostic(value: str) -> str:
+    bounded = _truncate_to_utf8_bytes(value, _DIAGNOSTIC_BYTE_LIMIT)
+    return bounded or "unreadable file"
+
+
+def _validate_sha256(value: str, field_name: str) -> str:
+    if not _SHA256_RE.fullmatch(value):
+        raise ValueError(f"{field_name} must be exactly 64 lowercase hex characters")
+    return value
+
+
+def _validate_diagnostics(value: tuple[str, ...]) -> tuple[str, ...]:
+    for diagnostic in value:
+        if not diagnostic.strip():
+            raise ValueError("diagnostics must be nonblank")
+        if len(diagnostic.encode("utf-8")) > _DIAGNOSTIC_BYTE_LIMIT:
+            raise ValueError("diagnostics may contain at most 2048 UTF-8 bytes")
+    return value
+
+
+class MillforgeBaseContextFile(BaseModel):
+    """One included private project-context record."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    scope: Literal["global", "project"]
+    path: Path
+    content: StrictStr
+    raw_sha256: StrictStr
+    original_byte_count: StrictInt = Field(ge=0)
+    included_byte_count: StrictInt = Field(ge=0)
+    truncated: StrictBool
+
+    @field_validator("raw_sha256")
+    @classmethod
+    def _raw_sha256_is_valid(cls, value: str) -> str:
+        return _validate_sha256(value, "raw_sha256")
+
+
+class MillforgeBaseContextSnapshot(BaseModel):
+    """The bounded, ordered private context available to prompt assembly."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    files: tuple[MillforgeBaseContextFile, ...]
+    diagnostics: tuple[StrictStr, ...]
+    context_sha256: StrictStr
+    truncated: StrictBool
+
+    @field_validator("diagnostics")
+    @classmethod
+    def _diagnostics_are_valid(cls, value: tuple[str, ...]) -> tuple[str, ...]:
+        return _validate_diagnostics(value)
+
+    @field_validator("context_sha256")
+    @classmethod
+    def _context_sha256_is_valid(cls, value: str) -> str:
+        return _validate_sha256(value, "context_sha256")
+
+
+@dataclass(frozen=True)
+class _DiscoveredContextFile:
+    scope: Literal["global", "project"]
+    path: Path
+    raw: bytes
+    content: str
+
+
+def _resolved_absolute_path(path: Path, field_name: str) -> Path:
+    if not path.is_absolute():
+        raise ValueError(f"{field_name} must be absolute")
+    return path.resolve()
+
+
+def _context_file_from_directory(
+    directory: Path,
+    scope: Literal["global", "project"],
+    diagnostics: list[str],
+) -> _DiscoveredContextFile | None:
+    attempted_candidates: list[Path] = []
+    for filename in _CONTEXT_CANDIDATES:
+        candidate = (directory / filename).resolve()
+        if not candidate.exists():
+            continue
+        if any(
+            _paths_refer_to_same_file(candidate, attempted)
+            for attempted in attempted_candidates
+        ):
+            continue
+        attempted_candidates.append(candidate)
+        try:
+            raw = candidate.read_bytes()
+        except OSError as error:
+            diagnostics.append(
+                _bounded_diagnostic(f"Could not read {candidate}: {error}")
+            )
+            continue
+        return _DiscoveredContextFile(
+            scope=scope,
+            path=candidate,
+            raw=raw,
+            content=raw.decode("utf-8", errors="replace"),
+        )
+    return None
+
+
+def _project_directories(cwd: Path) -> tuple[Path, ...]:
+    return tuple(reversed(cwd.parents)) + (cwd,)
+
+
+def _paths_refer_to_same_file(left: Path, right: Path) -> bool:
+    """Compare host filesystem identity, falling back across removal races."""
+
+    if left == right:
+        return True
+    try:
+        return left.samefile(right)
+    except OSError:
+        return False
+
+
+def _context_sha256(files: tuple[MillforgeBaseContextFile, ...]) -> str:
+    payload = [
+        {
+            "scope": file.scope,
+            "path": file.path.as_posix(),
+            "content": file.content,
+            "raw_sha256": file.raw_sha256,
+            "original_byte_count": file.original_byte_count,
+            "included_byte_count": file.included_byte_count,
+            "truncated": file.truncated,
+        }
+        for file in files
+    ]
+    canonical = json.dumps(
+        payload,
+        sort_keys=True,
+        ensure_ascii=True,
+        allow_nan=False,
+        separators=(",", ":"),
+    ).encode("ascii")
+    return hashlib.sha256(canonical).hexdigest()
+
+
+def _allocate_context(
+    discovered_files: tuple[_DiscoveredContextFile, ...],
+) -> tuple[tuple[MillforgeBaseContextFile, ...], bool]:
+    remaining = _CONTEXT_BYTE_LIMIT
+    included: list[MillforgeBaseContextFile] = []
+
+    for index, discovered in enumerate(discovered_files):
+        content_bytes = len(discovered.content.encode("utf-8"))
+        raw_sha256 = hashlib.sha256(discovered.raw).hexdigest()
+        if content_bytes <= remaining:
+            included.append(
+                MillforgeBaseContextFile(
+                    scope=discovered.scope,
+                    path=discovered.path,
+                    content=discovered.content,
+                    raw_sha256=raw_sha256,
+                    original_byte_count=len(discovered.raw),
+                    included_byte_count=content_bytes,
+                    truncated=False,
+                )
+            )
+            remaining -= content_bytes
+            if remaining == 0 and index < len(discovered_files) - 1:
+                return tuple(included), True
+            continue
+
+        content = _truncate_to_utf8_bytes(discovered.content, remaining)
+        included.append(
+            MillforgeBaseContextFile(
+                scope=discovered.scope,
+                path=discovered.path,
+                content=content,
+                raw_sha256=raw_sha256,
+                original_byte_count=len(discovered.raw),
+                included_byte_count=len(content.encode("utf-8")),
+                truncated=True,
+            )
+        )
+        return tuple(included), True
+
+    return tuple(included), False
+
+
+def load_millforge_base_context(
+    *, cwd: Path, home_directory: Path, enabled: bool
+) -> MillforgeBaseContextSnapshot:
+    """Load Pi-ordered context records under Millforge's finite byte bound."""
+
+    resolved_cwd = _resolved_absolute_path(cwd, "cwd")
+    resolved_home = _resolved_absolute_path(home_directory, "home_directory")
+    return _load_millforge_base_context_resolved(
+        cwd=resolved_cwd,
+        home_directory=resolved_home,
+        enabled=enabled,
+    )
+
+
+def _load_millforge_base_context_resolved(
+    *, cwd: Path, home_directory: Path, enabled: bool
+) -> MillforgeBaseContextSnapshot:
+    if not enabled:
+        empty_files: tuple[MillforgeBaseContextFile, ...] = ()
+        return MillforgeBaseContextSnapshot(
+            files=empty_files,
+            diagnostics=(),
+            context_sha256=_context_sha256(empty_files),
+            truncated=False,
+        )
+
+    diagnostics: list[str] = []
+    discovered: list[_DiscoveredContextFile] = []
+    native_global = _context_file_from_directory(
+        home_directory / ".millforge", "global", diagnostics
+    )
+    if native_global is None:
+        native_global = _context_file_from_directory(
+            home_directory / ".pi" / "agent", "global", diagnostics
+        )
+    if native_global is not None:
+        discovered.append(native_global)
+
+    seen_paths = [file.path for file in discovered]
+    for directory in _project_directories(cwd):
+        project_file = _context_file_from_directory(directory, "project", diagnostics)
+        if project_file is None or any(
+            _paths_refer_to_same_file(project_file.path, seen_path)
+            for seen_path in seen_paths
+        ):
+            continue
+        discovered.append(project_file)
+        seen_paths.append(project_file.path)
+
+    files, truncated = _allocate_context(tuple(discovered))
+    return MillforgeBaseContextSnapshot(
+        files=files,
+        diagnostics=tuple(diagnostics),
+        context_sha256=_context_sha256(files),
+        truncated=truncated,
+    )