graphcheck 0.1.0__py3-none-any.whl

graphcheck/contracts/results.py

@@ -0,0 +1,409 @@
+from __future__ import annotations
+
+from collections import Counter
+from datetime import datetime, timedelta
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    ValidationInfo,
+    field_validator,
+    model_validator,
+)
+
+from graphcheck.scoring import SEVERITY_WEIGHTS, calculate_score, calculate_suite_scores
+
+SCHEMA_VERSION = "1.2"
+
+
+class Verdict(StrEnum):
+    PASS = "pass"
+    FAIL = "fail"
+    WARN = "warn"
+    ERRORED = "errored"
+    SKIPPED = "skipped"
+
+
+class Severity(StrEnum):
+    ERROR = "error"
+    WARN = "warn"
+
+
+class Pattern(StrEnum):
+    CONFORMANCE = "conformance"
+    DRIFT = "drift"
+    COMPETENCY_SHAPE = "competency-shape"
+    COMPETENCY_REGRESSION = "competency-regression"
+
+
+class SkipReason(StrEnum):
+    GENERATED = "generated"
+    UNSUPPORTED = "unsupported"
+    NOT_RUN = "not_run"
+
+
+class RunStatus(StrEnum):
+    COMPLETE = "complete"
+    PARTIAL = "partial"
+    FAILED = "failed"
+
+
+class RedactionPolicy(StrEnum):
+    NONE = "none"
+    MASK = "mask"
+    HASH = "hash"
+
+
+WEIGHTS: dict[Severity, int] = {severity: SEVERITY_WEIGHTS[severity.value] for severity in Severity}
+
+
+def parse_utc_timestamp(value: str) -> datetime:
+    """Parse a frozen results timestamp and require an explicit UTC offset."""
+
+    try:
+        parsed = datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise ValueError("timestamp must be a valid ISO 8601 datetime") from exc
+    if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0):
+        raise ValueError("timestamp must include an explicit UTC offset")
+    return parsed
+
+
+class _Strict(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class EvidenceElement(_Strict):
+    kind: Literal["node", "rel", "aggregate"]
+    id: str
+    labels: list[str] | None = None
+    type: str | None = None
+
+
+class Evidence(_Strict):
+    message: str
+    elements: list[EvidenceElement] = Field(min_length=1)
+    truncated: bool
+    cap: int
+    total_count: int
+
+
+class Estimate(_Strict):
+    sample_size: int
+    population: int
+    confidence: float
+    ci: tuple[float, float] | None = None
+
+
+class CheckError(_Strict):
+    code: str
+    message: str
+    fix: str
+
+
+class CheckResult(_Strict):
+    # SPEC-01 freezes the shape: every key is present. The nullable/false keys carry
+    # null/false when unused (no defaults), so a producer that omits one fails validation.
+    id: str
+    suite_id: str
+    pattern: Pattern
+    name: str
+    provenance: str | None
+    severity: Severity
+    verdict: Verdict
+    skip_reason: SkipReason | None
+    started_at: str | None
+    duration_ms: int | None
+    compiled_query: str | None
+    params: dict[str, object] | None
+    measured: dict[str, object] | None
+    expected: dict[str, object]
+    estimate: Estimate | Literal[False]
+    evidence: Evidence | None
+    error: CheckError | None
+
+    @property
+    def executed(self) -> bool:
+        return self.verdict is not Verdict.SKIPPED
+
+    @model_validator(mode="after")
+    def _field_presence(self) -> CheckResult:
+        v = self.verdict
+        run_outcomes = (Verdict.PASS, Verdict.FAIL, Verdict.WARN)
+
+        # An assertion failure encodes its severity in the verdict (SPEC-01 rule 1): a fail is an
+        # error-severity failure, a warn is a warn-severity failure. Reject mismatches — otherwise
+        # exit_code could downgrade (e.g. severity:error + verdict:warn would exit 2, not 1).
+        if v is Verdict.FAIL and self.severity is not Severity.ERROR:
+            raise ValueError(f"fail check {self.id!r} must have severity:error")
+        if v is Verdict.WARN and self.severity is not Severity.WARN:
+            raise ValueError(f"warn check {self.id!r} must have severity:warn")
+
+        # Only an attempted, measured check can be sampled; errored/skipped are never estimates.
+        if v not in run_outcomes and self.estimate is not False:
+            raise ValueError(f"{v.value} check {self.id!r} must have estimate=false")
+
+        if v in (Verdict.FAIL, Verdict.WARN):
+            if self.evidence is None:
+                raise ValueError(f"{v.value} check {self.id!r} must carry evidence")
+        elif self.evidence is not None:
+            raise ValueError(f"{v.value} check {self.id!r} must not carry evidence")
+
+        if v is Verdict.ERRORED:
+            if self.error is None:
+                raise ValueError(f"errored check {self.id!r} must carry error")
+        elif self.error is not None:
+            raise ValueError(f"non-errored check {self.id!r} must not carry error")
+
+        if v is Verdict.SKIPPED:
+            if self.skip_reason is None:
+                raise ValueError(f"skipped check {self.id!r} must carry skip_reason")
+            for field in ("started_at", "duration_ms", "compiled_query", "params", "measured"):
+                if getattr(self, field) is not None:
+                    raise ValueError(f"skipped check {self.id!r} must have null {field}")
+            return self  # nothing executed; the checks below are for attempted checks
+
+        if self.skip_reason is not None:
+            raise ValueError(f"non-skipped check {self.id!r} must not carry skip_reason")
+
+        # Every attempted check (pass/fail/warn/errored) has timing.
+        if self.started_at is None or self.duration_ms is None:
+            raise ValueError(f"attempted check {self.id!r} must carry started_at and duration_ms")
+
+        if v in run_outcomes:
+            for field in ("compiled_query", "params", "measured"):
+                if getattr(self, field) is None:
+                    raise ValueError(f"{v.value} check {self.id!r} must carry {field}")
+        elif self.measured is not None:  # v is ERRORED here
+            raise ValueError(
+                f"errored check {self.id!r} must not carry measured (it did not measure)"
+            )
+        return self
+
+
+def score_value(checks: list[CheckResult]) -> int | None:
+    """Compatibility wrapper for the canonical scorer."""
+
+    return calculate_score(checks).value
+
+
+def totals(checks: list[CheckResult]) -> dict[str, int]:
+    counts = Counter(c.verdict for c in checks)
+    return {
+        "checks": len(checks),
+        "pass": counts[Verdict.PASS],
+        "fail": counts[Verdict.FAIL],
+        "warn": counts[Verdict.WARN],
+        "errored": counts[Verdict.ERRORED],
+        "skipped": counts[Verdict.SKIPPED],
+    }
+
+
+def exit_code(status: RunStatus, checks: list[CheckResult]) -> int:
+    if status is RunStatus.FAILED:
+        return 3
+    hard = any(
+        c.verdict is Verdict.FAIL
+        or (
+            c.verdict is Verdict.ERRORED
+            and c.severity is Severity.ERROR
+            and not (
+                status is RunStatus.PARTIAL
+                and c.error is not None
+                and c.error.code == "engine.timeout"
+            )
+        )
+        for c in checks
+    )
+    if hard:
+        return 1
+    nothing_evaluated = not any(c.executed for c in checks)
+    soft = any(
+        c.verdict is Verdict.WARN or (c.verdict is Verdict.ERRORED and c.severity is Severity.WARN)
+        for c in checks
+    )
+    if status is RunStatus.PARTIAL or nothing_evaluated or soft:
+        return 2
+    return 0
+
+
+class Totals(BaseModel):
+    # No populate_by_name: the external key is the frozen `pass` alias only, never `passed`.
+    model_config = ConfigDict(extra="forbid")
+    checks: int
+    passed: int = Field(alias="pass")
+    fail: int
+    warn: int
+    errored: int
+    skipped: int
+
+
+class Score(_Strict):
+    # Frozen shape: a non-null score is { value, method, weights } — all present, no defaults.
+    value: int
+    method: Literal["weighted-by-severity"]
+    weights: dict[str, int]
+
+    @model_validator(mode="after")
+    def _weights_locked(self) -> Score:
+        expected_weights = dict(SEVERITY_WEIGHTS)
+        if self.weights != expected_weights:
+            raise ValueError(f"score.weights are hard-coded in v0: {expected_weights}")
+        return self
+
+
+class Capabilities(_Strict):
+    apoc: bool
+    count_store: bool
+
+
+class RunTarget(_Strict):
+    database: str
+    server_version: str
+    edition: str
+    fingerprint: str
+    capabilities: Capabilities
+
+
+class ResultsTarget(RunTarget):
+    nodes: int | None = Field(default=None, ge=0)
+    relationships: int | None = Field(default=None, ge=0)
+    labels: list[str] | None
+    relationship_types: list[str] | None
+
+    @field_validator("labels", "relationship_types")
+    @classmethod
+    def _inventory_is_canonical(cls, value: list[str] | None) -> list[str] | None:
+        if value is not None and value != sorted(set(value)):
+            raise ValueError("target inventory must be sorted and contain no duplicates")
+        return value
+
+
+class Selection(_Strict):
+    suites: list[str]
+    tags: list[str]
+    fail_fast: bool
+
+
+class Redaction(_Strict):
+    policy: RedactionPolicy
+    applied: bool
+
+
+class Run(_Strict):
+    id: str
+    started_at: str
+    finished_at: str
+    graphcheck_version: str
+    pack_version: str
+    status: RunStatus
+    partial_reason: str | None  # present-but-nullable; non-null iff status is partial
+    exit_code: int
+    selection: Selection
+    redaction: Redaction
+    target: ResultsTarget | None  # present-but-nullable; null only for failed runs
+    error: CheckError | None  # present-but-nullable; non-null only for failed runs
+
+    @field_validator("started_at", "finished_at")
+    @classmethod
+    def _timestamps_are_utc(cls, value: str) -> str:
+        parse_utc_timestamp(value)
+        return value
+
+    @model_validator(mode="after")
+    def _timestamps_are_ordered(self) -> Run:
+        if parse_utc_timestamp(self.finished_at) < parse_utc_timestamp(self.started_at):
+            raise ValueError("finished_at must not precede started_at")
+        return self
+
+
+class Suite(_Strict):
+    id: str
+    source_sha: str
+    score: int | None
+    totals: Totals
+
+
+class Results(_Strict):
+    schema_version: Literal["1.2"]  # frozen top-level key, required and present
+    run: Run
+    score: Score | None
+    totals: Totals
+    suites: list[Suite]
+    checks: list[CheckResult]
+    _historical_schema_version: str | None = PrivateAttr(default=None)
+
+    @model_validator(mode="after")
+    def _consistency(self, info: ValidationInfo) -> Results:
+        status = self.run.status
+        if status is RunStatus.FAILED:
+            if self.run.error is None:
+                raise ValueError("failed run must carry run.error")
+            if self.checks or self.suites or self.score is not None:
+                raise ValueError("failed run must have empty checks/suites and null score")
+        elif self.run.error is not None:
+            raise ValueError("non-failed run must not carry run.error")
+
+        if status in (RunStatus.COMPLETE, RunStatus.PARTIAL) and self.run.target is None:
+            raise ValueError("complete/partial run must carry run.target")
+        historical = (info.context or {}).get("historical_schema_version") in {"1.0", "1.1"}
+        if (
+            self.run.target is not None
+            and (self.run.target.labels is None or self.run.target.relationship_types is None)
+            and not historical
+        ):
+            raise ValueError("schema 1.2 target inventory must contain non-null arrays")
+
+        if (self.run.partial_reason is not None) != (status is RunStatus.PARTIAL):
+            raise ValueError("partial_reason must be non-null iff status is partial")
+
+        expected_totals = totals(self.checks)
+        if self.totals.model_dump(by_alias=True) != expected_totals:
+            raise ValueError(f"totals must equal the tally of checks: {expected_totals}")
+
+        expected_score = calculate_score(self.checks).value
+        if expected_score is None:
+            if self.score is not None:
+                raise ValueError("score must be null when no check executed")
+        elif self.score is None or self.score.value != expected_score:
+            raise ValueError(f"score.value must be {expected_score}")
+
+        expected_exit = exit_code(status, self.checks)
+        if self.run.exit_code != expected_exit:
+            raise ValueError(f"exit_code must be {expected_exit}")
+
+        has_gap = any(
+            c.skip_reason in (SkipReason.UNSUPPORTED, SkipReason.NOT_RUN) for c in self.checks
+        )
+        if has_gap and status is not RunStatus.PARTIAL:
+            raise ValueError("an unsupported/not_run skip requires run.status:partial")
+
+        identities = [(c.suite_id, c.id) for c in self.checks]
+        if len(identities) != len(set(identities)):
+            raise ValueError("check identity (suite_id, id) must be unique across checks[]")
+        if len({s.id for s in self.suites}) != len(self.suites):
+            raise ValueError("suite ids must be unique in suites[]")
+
+        by_suite: dict[str, list[CheckResult]] = {}
+        for c in self.checks:
+            by_suite.setdefault(c.suite_id, []).append(c)
+        suite_ids = {s.id for s in self.suites}
+        for suite_id in by_suite:
+            if suite_id not in suite_ids:
+                raise ValueError(f"check suite_id {suite_id!r} has no matching suites[] entry")
+        suite_scores = calculate_suite_scores(self.checks)
+        for suite in self.suites:
+            members = by_suite.get(suite.id, [])
+            expected_suite_score = suite_scores.get(suite.id)
+            expected_suite_value = (
+                None if expected_suite_score is None else expected_suite_score.value
+            )
+            if suite.score != expected_suite_value:
+                raise ValueError(f"suite {suite.id!r} score is inconsistent with its checks")
+            if suite.totals.model_dump(by_alias=True) != totals(members):
+                raise ValueError(f"suite {suite.id!r} totals are inconsistent with its checks")
+        return self

graphcheck/contracts/scalars.py

@@ -0,0 +1,23 @@
+from typing import Annotated
+
+from pydantic import BeforeValidator, Field
+
+
+def _normalize_json_schema_integer(value: object) -> object:
+    """Match Draft 2020-12 integer semantics without general coercion."""
+    if type(value) is float and value.is_integer():
+        return int(value)
+    return value
+
+
+JsonSchemaInteger = Annotated[int, BeforeValidator(_normalize_json_schema_integer)]
+NonNegativeJsonSchemaInteger = Annotated[
+    int,
+    Field(ge=0),
+    BeforeValidator(_normalize_json_schema_integer),
+]
+PositiveJsonSchemaInteger = Annotated[
+    int,
+    Field(gt=0),
+    BeforeValidator(_normalize_json_schema_integer),
+]

graphcheck/contracts/schemas.py

@@ -0,0 +1,140 @@
+import json
+from pathlib import Path
+
+from pydantic import TypeAdapter
+
+from graphcheck.contracts.check import _SuiteFile
+from graphcheck.contracts.profile import BaselineProfile
+from graphcheck.contracts.results import Results
+from graphcheck.packs import PACK_VERSION, REGISTRY
+from graphcheck.packs.metadata import PackMetadata
+
+SPECS_DIR = Path(__file__).resolve().parents[3] / "docs" / "specs"
+JSON_SCHEMA_2020_12 = "https://json-schema.org/draft/2020-12/schema"
+_CHECK_SCHEMA_COMMENT = (
+    "Portable structural contract. Consumers must additionally enforce the semantic "
+    "invariants in SPEC-02: globally unique check ids, distinct label_cooccurrence "
+    "labels, distinct rel_direction endpoint labels, and distinct temporal_sanity "
+    "properties. GraphCheck enforces them in load_suite()."
+)
+
+
+def results_schema() -> dict:
+    return Results.model_json_schema(by_alias=True)
+
+
+def profile_schema() -> dict:
+    return BaselineProfile.model_json_schema()
+
+
+def write_profile_schema() -> Path:
+    SPECS_DIR.mkdir(parents=True, exist_ok=True)
+    path = SPECS_DIR / "profile.schema.json"
+    path.write_text(json.dumps(profile_schema(), indent=2, sort_keys=True) + "\n")
+    return path
+
+
+def write_results_schema() -> Path:
+    SPECS_DIR.mkdir(parents=True, exist_ok=True)
+    path = SPECS_DIR / "results.schema.json"
+    path.write_text(json.dumps(results_schema(), indent=2, sort_keys=True) + "\n")
+    return path
+
+
+def pack_metadata_schema() -> dict:
+    schema = TypeAdapter(PackMetadata).json_schema()
+    schema["$schema"] = JSON_SCHEMA_2020_12
+    schema["x-pack-version"] = PACK_VERSION
+    return schema
+
+
+def validate_pack_metadata_schema(instance: object) -> None:
+    """Validate pack metadata, including standard JSON Schema format assertions."""
+    from jsonschema import Draft202012Validator, FormatChecker
+
+    schema = pack_metadata_schema()
+    Draft202012Validator.check_schema(schema)
+    validator = Draft202012Validator(
+        schema,
+        format_checker=FormatChecker(),
+    )
+    validator.validate(instance)
+
+
+def write_pack_metadata_schema() -> Path:
+    SPECS_DIR.mkdir(parents=True, exist_ok=True)
+    path = SPECS_DIR / "pack.schema.json"
+    path.write_text(json.dumps(pack_metadata_schema(), indent=2, sort_keys=True) + "\n")
+    return path
+
+
+def check_envelope_schema() -> dict:
+    schema = _SuiteFile.model_json_schema(by_alias=True)
+    schema["$schema"] = JSON_SCHEMA_2020_12
+    schema["$comment"] = (
+        "Frozen SPEC-02 envelope only; use check.schema.json for pack-owned `with` validation."
+    )
+    return schema
+
+
+def check_combined_schema() -> dict:
+    schema = _SuiteFile.model_json_schema(by_alias=True)
+    schema["$schema"] = JSON_SCHEMA_2020_12
+    schema["$comment"] = _CHECK_SCHEMA_COMMENT
+    schema["x-pack-version"] = PACK_VERSION
+    defs = schema["$defs"]
+    # Constrain each conformance item's `with` by its `check`: move the auto-generated
+    # ConformanceCheck def aside, then redefine its $ref target as an allOf of the base
+    # plus a oneOf over the registry. The conformance array already $refs ConformanceCheck,
+    # so it picks up the constraint without touching the array schema.
+    defs["ConformanceCheckBase"] = defs["ConformanceCheck"]
+    branches = []
+    for name, model in sorted(REGISTRY.items()):
+        with_schema = model.model_json_schema()
+        if "$defs" in with_schema:
+            # A nested pack model emits internal #/$defs/... refs that dangle once inlined here.
+            # v0 pack `with` models must be flat/ref-free; hoisting $defs is C3's job when needed.
+            raise ValueError(
+                f"pack `with` model {name!r} emits $defs; v0 requires flat, ref-free pack "
+                f"schemas so they can be inlined. Hoist its $defs before registering it."
+            )
+        branches.append(
+            {
+                "properties": {"check": {"const": name}, "with": with_schema},
+                "required": ["check", "with"],
+            }
+        )
+    defs["WithByCheck"] = {"oneOf": branches}
+    defs["ConformanceCheck"] = {
+        "allOf": [
+            {"$ref": "#/$defs/ConformanceCheckBase"},
+            {"$ref": "#/$defs/WithByCheck"},
+        ]
+    }
+    return schema
+
+
+def validate_check_schema(instance: object) -> None:
+    """Validate the portable structural schema, including format assertions.
+
+    Use ``load_suite`` for the additional semantic invariants defined by SPEC-02.
+    """
+    from jsonschema import Draft202012Validator, FormatChecker
+
+    schema = check_combined_schema()
+    Draft202012Validator.check_schema(schema)
+    validator = Draft202012Validator(
+        schema,
+        format_checker=FormatChecker(),
+    )
+    validator.validate(instance)
+
+
+def write_check_schemas() -> None:
+    SPECS_DIR.mkdir(parents=True, exist_ok=True)
+    (SPECS_DIR / "check.envelope.schema.json").write_text(
+        json.dumps(check_envelope_schema(), indent=2, sort_keys=True) + "\n"
+    )
+    (SPECS_DIR / "check.schema.json").write_text(
+        json.dumps(check_combined_schema(), indent=2, sort_keys=True) + "\n"
+    )