harness-common 0.3.1__py3-none-any.whl

harness_common/__init__.py

@@ -0,0 +1,57 @@
+"""common — harness-neutral shared code and runtime identities.
+
+The harnesses (e2e / eval / perf / trace / trajectory) depend on this package but never on each
+other; only genuinely neutral code lives here (no harness domain concept leaks in).
+
+The canonical case model — `Case` / `CaseSet` / `FacetSchema` and the `Face` / `FACES`
+judgment-face enum — lives upstream in `spec_case.model` / `spec_case.facets`
+(the spec-case package, `[model]` extra): the case format is the asset layer's contract,
+and this repo is one of its runners. Import those names from `spec_case` directly.
+
+`Forge` and `Repository` identify where source lives; `Product` names a business product;
+`Component` identifies a buildable unit within a Repository. `Environment` identifies where
+execution happens. `Service` identifies a Component's runtime presence in one Environment.
+`Workload` locates a declared runtime carrier; a Service can reference multiple workloads.
+`Operation` identifies a capability exposed by a Service; `HttpOperation` adds its HTTP
+transport contract. An `Execution` groups domain-defined work such as an e2e CaseRun or
+a perf Trial; each `OperationRun` owns one raw `Outcome`. Domain engines own scheduling
+details. A domain `Reducer` projects the recorded Outcomes into one or more
+`Artifact` objects without calling the tested Service again. `Deployment` records an
+attempt applied through a `Deployer`.
+`Experiment` names reproducible verification intent; `ExperimentRun` records one execution
+and references its `Artifact` outputs.
+"""
+
+from __future__ import annotations
+
+from harness_common.artifact import Artifact as Artifact
+from harness_common.component import Component as Component
+from harness_common.deployment import Deployer as Deployer
+from harness_common.deployment import Deployment as Deployment
+from harness_common.environment import Environment as Environment
+from harness_common.environment import KubernetesEnvironment as KubernetesEnvironment
+from harness_common.environment import HostEnvironment as HostEnvironment
+from harness_common.host import Host as Host
+from harness_common.environment import EnvironmentFacts as EnvironmentFacts
+from harness_common.environment import EnvironmentSnapshot as EnvironmentSnapshot
+from harness_common.execution import Execution as Execution
+from harness_common.experiment import Experiment as Experiment
+from harness_common.experiment import ExperimentRun as ExperimentRun
+from harness_common.forge import Forge as Forge
+from harness_common.operation import HttpOperation as HttpOperation
+from harness_common.operation import Operation as Operation
+from harness_common.operation import OperationRun as OperationRun
+from harness_common.outcome import Outcome as Outcome
+from harness_common.product import Product as Product
+from harness_common.repository import Repository as Repository
+from harness_common.reducer import Reducer as Reducer
+from harness_common.service import Service as Service
+from harness_common.workload import KubernetesWorkload as KubernetesWorkload
+from harness_common.workload import Workload as Workload
+
+from harness_common.client import Client as Client
+from harness_common.client import ClientManager as ClientManager
+from harness_common.client import ClientProvider as ClientProvider
+from harness_common.client import DataSource as DataSource
+from harness_common.client import ServiceDataSource as ServiceDataSource
+from harness_common.client import data_source_key as data_source_key

harness_common/artifact.py

@@ -0,0 +1,17 @@
+"""A durable output produced by an ExperimentRun."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Artifact:
+    """A named domain result referenced relative to its run directory.
+
+    The producing harness owns the content and schema. Reports render one or more Artifacts
+    without becoming a new source of execution facts.
+    """
+
+    name: str
+    path: str

harness_common/client.py

@@ -0,0 +1,136 @@
+"""Execution-scoped, asynchronous client ownership.
+
+DataSource keys describe configuration, not object identity. Consumers receive a
+ClientProvider; only the root execution owns and disposes the ClientManager.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Generic, Protocol, TypeVar, cast
+
+from harness_common.service import Service
+
+
+class Client(Protocol):
+    async def initialize(self) -> None: ...
+    async def dispose(self) -> None: ...
+
+
+C = TypeVar("C", bound=Client, covariant=True)
+
+
+class DataSource(Protocol[C]):
+    @property
+    def key(self) -> str: ...
+    def create_client(self, clients: ClientProvider) -> C: ...
+
+
+@dataclass(frozen=True)
+class ServiceDataSource(Generic[C]):
+    """One access association, not a platform resource or a client owner.
+
+    A Service can have zero or many associations; the same source can be shared
+    by multiple Services. Platform workload mappings belong to deployment config.
+    Use clients.get(binding.source) so logical identity never fragments client reuse.
+    """
+
+    service: Service
+    source: DataSource[C]
+
+
+class ClientProvider(Protocol):
+    async def get(self, source: DataSource[C]) -> C: ...
+
+
+def data_source_key(protocol: str, configuration: object) -> str:
+    """Hash JSON-compatible configuration, including credentials, without exposing it."""
+    canonical = json.dumps(
+        configuration, sort_keys=True, separators=(",", ":"), allow_nan=False
+    )
+    return protocol + ":" + hashlib.sha256(canonical.encode()).hexdigest()
+
+
+class ClientManager:
+    """Share initialized clients within one event loop and close them at root exit.
+
+    Initialize dependencies through the supplied provider before publishing a
+    client as ready. Reverse readiness order then closes consumers first.
+    Join all command work before leaving the root context: borrowed clients are
+    invalid after disposal. Managers cannot be reopened or shared across loops.
+    """
+
+    def __init__(self) -> None:
+        self._pending: dict[str, asyncio.Task[Client]] = {}
+        self._ready: list[Client] = []
+        self._disposal: asyncio.Task[None] | None = None
+        self._cleanup_errors: list[BaseException] = []
+        self._unclean: set[str] = set()
+
+    async def get(self, source: DataSource[C]) -> C:
+        if self._disposal is not None:
+            raise RuntimeError("client manager is disposed")
+        key = source.key
+        task = self._pending.get(key)
+        if task is None:
+            # Store before the task can run; factories can themselves request dependencies.
+            task = asyncio.create_task(self._initialize(source))
+            self._pending[key] = task
+            task.add_done_callback(lambda done: self._settled(key, done))
+        # A cancelled consumer must not cancel initialization needed by other consumers.
+        return cast(C, await asyncio.shield(task))
+
+    def _settled(self, key: str, task: asyncio.Task[Client]) -> None:
+        failed = task.cancelled() or task.exception() is not None
+        if failed and key not in self._unclean and self._pending.get(key) is task:
+            del self._pending[key]
+
+    async def _initialize(self, source: DataSource[Client]) -> Client:
+        client = source.create_client(self)
+        try:
+            await client.initialize()
+            self._ready.append(client)
+            return client
+        except BaseException as error:
+            try:
+                await client.dispose()
+            except BaseException as cleanup:
+                self._cleanup_errors.append(cleanup)
+                self._unclean.add(source.key)
+                raise BaseExceptionGroup(
+                    "client initialization and cleanup failed", [error, cleanup]
+                ) from None
+            raise
+
+    async def dispose(self) -> None:
+        """Idempotent, concurrent-safe cleanup; cancellation of a waiter cannot stop it."""
+        if self._disposal is None:
+            self._disposal = asyncio.create_task(self._dispose())
+        await asyncio.shield(self._disposal)
+
+    async def _dispose(self) -> None:
+        pending = list(self._pending.values())
+        for task in pending:
+            if not task.done():
+                task.cancel()
+        await asyncio.gather(*pending, return_exceptions=True)
+        self._pending.clear()
+        for client in reversed(self._ready):
+            try:
+                await client.dispose()
+            except BaseException as error:
+                self._cleanup_errors.append(error)
+        self._ready.clear()
+        if self._cleanup_errors:
+            raise BaseExceptionGroup("client cleanup failed", self._cleanup_errors)
+
+    async def __aenter__(self) -> ClientManager:
+        if self._disposal is not None:
+            raise RuntimeError("client manager is disposed")
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        await self.dispose()

harness_common/component.py

@@ -0,0 +1,15 @@
+"""Stable identity of one buildable component within a code repository."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from harness_common.repository import Repository
+
+
+@dataclass(frozen=True, slots=True)
+class Component:
+    """A named component owned by a repository; one repository may own many."""
+
+    repository: Repository
+    name: str

harness_common/deployment.py

@@ -0,0 +1,31 @@
+"""Deployment records and the port that applies them."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+from harness_common.service import Service
+
+
+@dataclass(frozen=True, slots=True)
+class Deployment:
+    """One attempt to create or update a Service from its Component."""
+
+    service: Service
+
+
+DeploymentT = TypeVar("DeploymentT", bound=Deployment)
+
+
+class Deployer(ABC, Generic[DeploymentT]):
+    """Port implemented by a concrete deployment mechanism."""
+
+    @abstractmethod
+    async def deploy(self, deployment: DeploymentT) -> None:
+        """Apply one Deployment and return after the Service is ready."""
+
+    async def teardown(self) -> None:
+        """Restore the baseline when the concrete deployer supports it."""
+        return None

harness_common/environment.py

@@ -0,0 +1,97 @@
+"""Deployment environment identity, optional access host and observed evidence."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+from harness_common.host import Host, parse_host
+
+
+@dataclass(frozen=True, slots=True)
+class Environment:
+    """A named environment in which Components have running Service instances.
+
+    Host is optional. When omitted, operations use the current execution host.
+    It does not identify a Kubernetes node or dictate where the case runner runs.
+    """
+
+    name: str
+    host: Host | None = field(default=None, kw_only=True)
+    kind: ClassVar[str] = "generic"
+
+
+@dataclass(frozen=True, slots=True)
+class HostEnvironment(Environment):
+    """A target running directly on the environment's host."""
+
+    kind: ClassVar[str] = "host"
+
+
+@dataclass(frozen=True, slots=True)
+class KubernetesEnvironment(Environment):
+    """A cluster accessed from Host; kubeconfig is a path on that host."""
+
+    kubeconfig: str = ""
+    context: str | None = None
+    kind: ClassVar[str] = "kubernetes"
+
+
+@dataclass
+class EnvironmentFacts:
+    """Observed facts with provenance. Missing values mean unknown, not false."""
+
+    source: str = ""
+    observed_at: str = ""
+    values: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class EnvironmentSnapshot:
+    """Run evidence; access configuration and credentials are deliberately absent.
+
+    Runner and target facts are separate: a local runner can test a remote Pod.
+    Profiles select project-owned conditions and never substitute for observation.
+    """
+
+    name: str
+    kind: str
+    profile: str = "default"
+    revision: str = ""
+    host_name: str = ""
+    host: EnvironmentFacts | None = None
+    runner: EnvironmentFacts = field(default_factory=EnvironmentFacts)
+    target: EnvironmentFacts = field(default_factory=EnvironmentFacts)
+
+
+def parse_environment(data: dict) -> Environment:
+    """Parse the shared environment wire format, including legacy Kubernetes input."""
+    if not isinstance(data, dict):
+        raise ValueError("environment must be a mapping")
+    unknown = set(data) - {"name", "kind", "host", "kubeconfig", "context"}
+    if unknown:
+        raise ValueError(
+            f"environment contains unknown field(s): {', '.join(sorted(unknown))}"
+        )
+    if any(
+        value is not None and not isinstance(value, str)
+        for key, value in data.items()
+        if key != "host"
+    ):
+        raise ValueError("environment fields except host must be strings")
+    kind = data.get("kind") or (
+        "kubernetes" if data.get("kubeconfig") or data.get("context") else "generic"
+    )
+    if kind not in {"generic", "host", "kubernetes"}:
+        raise ValueError(f"unknown environment kind {kind!r}")
+    host = parse_host(data["host"]) if data.get("host") is not None else None
+    name = data.get("name") or ""
+    if kind == "kubernetes":
+        return KubernetesEnvironment(
+            name, data.get("kubeconfig") or "", data.get("context"), host=host
+        )
+    if data.get("kubeconfig") or data.get("context"):
+        raise ValueError(f"{kind} environment contains Kubernetes access fields")
+    if kind == "host":
+        return HostEnvironment(name, host=host)
+    return Environment(name, host=host)

harness_common/execution.py

@@ -0,0 +1,25 @@
+"""Harness-neutral grouping of related OperationRuns."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Generic, TypeVar
+
+from harness_common.operation import OperationRun
+from harness_common.outcome import Outcome
+
+
+OutcomeT = TypeVar("OutcomeT", bound=Outcome)
+
+
+@dataclass(kw_only=True)
+class Execution(Generic[OutcomeT]):
+    """One domain-defined unit of work within an ExperimentRun.
+
+    E2E and perf intentionally organize Operations differently.  The shared
+    model records only the stable result: an identity and the OperationRuns
+    grouped by that unit of work.
+    """
+
+    id: str
+    operation_runs: list[OperationRun[OutcomeT]] = field(default_factory=list)

harness_common/experiment.py

@@ -0,0 +1,60 @@
+"""Harness-neutral experiment identity and run artifacts."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from collections.abc import Iterator
+
+from harness_common.artifact import Artifact
+from harness_common.execution import Execution
+from harness_common.operation import OperationRun
+from harness_common.outcome import Outcome
+
+
+@dataclass(kw_only=True)
+class Experiment:
+    """A named, reproducible verification intent.
+
+    Domain harnesses add what varies and how it is executed. The shared concept owns only
+    the stable name used to group runs.
+    """
+
+    name: str
+
+
+@dataclass
+class ExperimentRun:
+    """One execution of an Experiment and the outputs it produced.
+
+    ``experiment`` remains the Experiment name on the persisted boundary so a run artifact
+    does not embed the complete, domain-specific experiment object.
+    """
+
+    run_id: str
+    experiment: str
+    created_at: str
+    executions: list[Execution] = field(default_factory=list, kw_only=True)
+    artifacts: list[Artifact] = field(default_factory=list, kw_only=True)
+
+    def operation_runs(self) -> Iterator[OperationRun]:
+        """Iterate all OperationRuns without duplicating the stored hierarchy."""
+        for execution in self.executions:
+            yield from execution.operation_runs
+
+    def outcomes(self) -> Iterator[Outcome]:
+        """Iterate raw Outcomes in execution order."""
+        for operation_run in self.operation_runs():
+            yield operation_run.outcome
+
+    def add_artifact(self, name: str, path: str) -> None:
+        """Register or replace one artifact by its logical name."""
+        artifact = Artifact(name=name, path=path)
+        for index, current in enumerate(self.artifacts):
+            if current.name == name:
+                self.artifacts[index] = artifact
+                return
+        self.artifacts.append(artifact)
+
+    def artifact_paths(self) -> dict[str, str]:
+        """Project typed artifacts to the cross-harness verdict wire shape."""
+        return {artifact.name: artifact.path for artifact in self.artifacts}

harness_common/forge.py

@@ -0,0 +1,12 @@
+"""Stable identity of a source-code forge."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Forge:
+    """A system that hosts source-code repositories."""
+
+    name: str

harness_common/host.py

@@ -0,0 +1,45 @@
+"""Host identity and access configuration; operations belong to toolbox."""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Host:
+    """An optional constituent of an Environment.
+
+    For a host environment this machine hosts the target. For Kubernetes it
+    provides cluster access, and kubeconfig paths refer to its filesystem.
+    SSH addresses use the caller's SSH configuration; credentials are not stored.
+    """
+
+    name: str
+    transport: str = "local"
+    address: str = ""
+
+    def validate(self) -> None:
+        if not self.name:
+            raise ValueError("host requires name")
+        if self.transport in {"", "local"}:
+            if self.address:
+                raise ValueError("local host cannot specify SSH address")
+        elif self.transport == "ssh":
+            if not self.address or self.address.startswith("-"):
+                raise ValueError("SSH host requires a non-option address")
+        else:
+            raise ValueError(f"unknown host transport {self.transport!r}")
+
+
+def parse_host(data: dict) -> Host:
+    """Parse the shared Host wire representation."""
+    if not isinstance(data, dict):
+        raise ValueError("host must be a mapping")
+    unknown = set(data) - {"name", "transport", "address"}
+    if unknown:
+        raise ValueError(
+            f"host contains unknown field(s): {', '.join(sorted(unknown))}"
+        )
+    if any(not isinstance(value, str) for value in data.values()):
+        raise ValueError("host fields must be strings")
+    host = Host(**data) if "name" in data else Host(name="", **data)
+    host.validate()
+    return host

harness_common/llm/__init__.py

@@ -0,0 +1,11 @@
+"""common.llm — OpenAI-compatible LLM client, harness-neutral shared infrastructure.
+
+A model call (endpoint / auth / timeout / retry / provenance) is not specific to any one
+harness: eval's LLM-as-judge metrics use it today, and e2e's casegen ``LLMCompiler`` will use
+it tomorrow. So it lives in ``common`` (like ``case`` / ``verdict`` / ``facets``) — the 3 SDKs
+compose ``LLMClient`` without importing each other.
+"""
+
+from harness_common.llm.client import ChatResult as ChatResult
+from harness_common.llm.client import LLMClient as LLMClient
+from harness_common.llm.client import LLMConfig as LLMConfig

harness_common/llm/client.py

@@ -0,0 +1,165 @@
+"""OpenAI-compatible LLM client with retry — shared by LLM-backed components.
+
+Owns the "call a model" concern (endpoint / auth / timeout / retry / provenance)
+so LLM-as-judge metrics and any future LLM-backed component (multi-aspect
+judges, LLM dataset builders, ...) **compose** an ``LLMClient`` instead of
+re-implementing HTTP + retry per metric.
+
+Layering:
+- ``LLMConfig``  — endpoint + key + model + timeout + retry knobs (``from_env``)
+- ``ChatResult`` — assistant text + ``model`` provenance (which model answered)
+- ``LLMClient``  — the call: OpenAI-compatible ``/chat/completions`` + retry,
+  with an optional shared ``asyncio.Semaphore`` to cap in-flight concurrency
+  (e.g. many judges × many samples hitting one endpoint → avoid 429s)
+
+Robustness is split by layer: the client *raises* on persistent failure;
+callers (e.g. ``BaseLLMJudge.score``) decide how to degrade (judge → None).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from dataclasses import dataclass
+
+import httpx
+
+
+@dataclass(frozen=True)
+class LLMConfig:
+    """Endpoint + key + model for an OpenAI-compatible chat completion API.
+
+    Resolve from env via ``from_env``; pass empty values explicitly for tests.
+    The default env var names (``EVAL_JUDGE_BASE`` / ``_KEY`` / ``_MODEL``) keep
+    LLM secrets out of yaml; pass other names for non-judge uses.
+    """
+
+    base_url: str
+    api_key: str
+    model: str
+    timeout_s: float = 60.0
+    # 瞬时失败重试：并发调模型时偶发超时 / 429 / 5xx 不应让调用方直接失败。
+    # 总尝试 = max_retries + 1。
+    max_retries: int = 2
+    retry_backoff_s: float = 1.0  # 指数退避基数：第 i 次重试前 sleep backoff * 2**i
+
+    @classmethod
+    def from_env(
+        cls,
+        *,
+        base_env: str = "EVAL_JUDGE_BASE",
+        key_env: str = "EVAL_JUDGE_KEY",
+        model_env: str = "EVAL_JUDGE_MODEL",
+        timeout_s: float = 60.0,
+        max_retries: int = 2,
+        retry_backoff_s: float = 1.0,
+    ) -> LLMConfig:
+        return cls(
+            base_url=os.environ.get(base_env, "").rstrip("/"),
+            api_key=os.environ.get(key_env, ""),
+            model=os.environ.get(model_env, ""),
+            timeout_s=timeout_s,
+            max_retries=max_retries,
+            retry_backoff_s=retry_backoff_s,
+        )
+
+    def ready(self) -> bool:
+        return bool(self.base_url and self.api_key and self.model)
+
+
+@dataclass(frozen=True)
+class ChatResult:
+    """One completion: the assistant text + which model produced it (provenance)."""
+
+    text: str
+    model: str
+
+
+class LLMClient:
+    """Async OpenAI-compatible chat client with retry on transient failures.
+
+    Construct one and share it across metrics to reuse config and (optionally) a
+    concurrency cap. Retries timeouts / network errors / 429 / 5xx up to
+    ``config.max_retries`` times with exponential backoff; other 4xx are not
+    retried (they won't fix themselves). On exhaustion the last error raises.
+    """
+
+    def __init__(
+        self, config: LLMConfig, *, semaphore: asyncio.Semaphore | None = None
+    ):
+        self.config = config
+        # 可选全局并发闸：多个 judge × 多 sample 并发判分时，限制对单个端点的在途请求数。
+        self._semaphore = semaphore
+
+    @classmethod
+    def from_env(
+        cls, *, semaphore: asyncio.Semaphore | None = None, **env_kwargs
+    ) -> LLMClient:
+        return cls(LLMConfig.from_env(**env_kwargs), semaphore=semaphore)
+
+    def ready(self) -> bool:
+        return self.config.ready()
+
+    async def complete(self, system: str, user: str) -> ChatResult:
+        """Convenience: a single system+user turn."""
+        return await self.chat(
+            [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ]
+        )
+
+    async def chat(self, messages: list[dict]) -> ChatResult:
+        """Call ``/chat/completions`` with retry. Raises on persistent failure."""
+        attempts = max(1, self.config.max_retries + 1)
+        async with httpx.AsyncClient(timeout=self.config.timeout_s) as client:
+            for i in range(attempts):
+                try:
+                    return await self._post(client, messages)
+                except Exception as exc:
+                    if i + 1 < attempts and _is_retryable(exc):
+                        await asyncio.sleep(self.config.retry_backoff_s * (2**i))
+                        continue
+                    raise
+        raise RuntimeError("unreachable")  # pragma: no cover
+
+    async def _post(
+        self, client: httpx.AsyncClient, messages: list[dict]
+    ) -> ChatResult:
+        """A single attempt. The optional semaphore caps concurrent in-flight
+        calls (released during backoff so retries don't hog slots)."""
+        if self._semaphore is not None:
+            async with self._semaphore:
+                return await self._post_once(client, messages)
+        return await self._post_once(client, messages)
+
+    async def _post_once(
+        self, client: httpx.AsyncClient, messages: list[dict]
+    ) -> ChatResult:
+        resp = await client.post(
+            f"{self.config.base_url}/chat/completions",
+            headers={
+                "Authorization": f"Bearer {self.config.api_key}",
+                "Content-Type": "application/json",
+            },
+            json={
+                "model": self.config.model,
+                "messages": messages,
+                "temperature": 0,
+            },
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        text = data["choices"][0]["message"]["content"]
+        return ChatResult(text=text, model=data.get("model") or self.config.model)
+
+
+def _is_retryable(exc: Exception) -> bool:
+    """Transient errors worth retrying: timeouts, network/transport errors,
+    429, and 5xx. Other 4xx (bad request / auth) won't fix on retry."""
+    if isinstance(exc, httpx.TimeoutException | httpx.TransportError):
+        return True
+    if isinstance(exc, httpx.HTTPStatusError):
+        code = exc.response.status_code
+        return code == 429 or code >= 500
+    return False