eval-toolkit 0.27.1__py3-none-any.whl

eval_toolkit/schemas/manifest.v2.json

@@ -0,0 +1,186 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://eval-toolkit/schemas/manifest.v2.json",
+  "version": "2",
+  "title": "eval-toolkit RunManifest (v2)",
+  "description": "Schema for run_dir/manifest.json. Produced by eval_toolkit.manifest.write_manifest. NeurIPS Reproducibility Checklist-aligned. v2 adds: captured_at (auto-populated ISO-8601 UTC); top-level data_revisions and metadata (replacing meta:/hf_*: prefixed extra_code_versions); tightened gpu_info typing (count: integer, memory_gb: number).",
+  "type": "object",
+  "required": ["schema_version", "run_id", "captured_at", "code_versions", "env"],
+  "properties": {
+    "schema_version": {
+      "const": "v2"
+    },
+    "run_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "captured_at": {
+      "type": "string",
+      "description": "ISO-8601 UTC timestamp captured at build_manifest() call time. Replaces meta:run_timestamp_utc abuse of extra_code_versions in v1 consumers.",
+      "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+    },
+    "git_sha": {
+      "type": ["string", "null"]
+    },
+    "dirty_flag": {
+      "type": "boolean",
+      "description": "True iff the working tree had uncommitted changes at run time (NeurIPS clean-replay concern)."
+    },
+    "code_versions": {
+      "type": "object",
+      "description": "{package_name: version_string} — at minimum eval_toolkit. Caller-declared key dependencies (e.g. transformers, torch, sklearn). For dataset / model revisions use data_revisions; for run-time metadata use metadata.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "data_revisions": {
+      "type": "object",
+      "description": "{logical_name: revision_string} for input dataset and model revisions. v2 addition (was crammed under extra_code_versions in v1). Recommended key prefixes: 'hf_dataset:<name>' and 'hf_model:<name>'.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "description": "{label_name: label_value} for caller-supplied run-time labels (cli_args, environment selectors, etc.). v2 addition (was crammed under extra_code_versions in v1).",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "seeds": {
+      "type": "object",
+      "description": "{source: seed} for global / bootstrap / torch / dataloader RNGs.",
+      "additionalProperties": {
+        "type": "integer"
+      }
+    },
+    "data_hashes": {
+      "type": "object",
+      "description": "{logical_name: 'sha256:...'} for input artifacts.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "config_hash": {
+      "type": "string",
+      "description": "SHA-256 over the canonical-JSON encoding of the eval config."
+    },
+    "env": {
+      "type": "object",
+      "description": "Environment fingerprint (python, platform, key dep versions).",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "gpu_info": {
+      "type": "object",
+      "description": "{name, count, memory_gb} from nvidia-smi (empty dict if unavailable). v2 tightens count to integer and memory_gb to number.",
+      "properties": {
+        "name": {"type": "string"},
+        "count": {"type": "integer", "minimum": 0},
+        "memory_gb": {"type": "number", "minimum": 0}
+      },
+      "additionalProperties": true
+    },
+    "cuda_version": {
+      "type": ["string", "null"]
+    },
+    "wall_clock_seconds": {
+      "type": ["number", "null"]
+    },
+    "versioned_objects": {
+      "type": "object",
+      "description": "{name: version} for any Tier-2 implementation exposing a `version` attribute.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "leakage_report": {
+      "type": ["object", "null"],
+      "description": "Serialized LeakageReport (or null if no leakage checks ran).",
+      "properties": {
+        "findings": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["check_name", "severity", "drop_indices", "evidence", "message", "n_affected"],
+            "properties": {
+              "check_name": {"type": "string"},
+              "severity": {"enum": ["error", "warning", "info"]},
+              "drop_indices": {"type": ["object", "null"]},
+              "evidence": {"type": "object"},
+              "message": {"type": "string"},
+              "n_affected": {"type": "integer", "minimum": 0}
+            }
+          }
+        }
+      }
+    },
+    "source_roles": {
+      "type": "array",
+      "description": "Optional generic source-role metadata for data-quality and evidence audits. Recommended roles: train, calibration, locked_eval, external_diagnostic, excluded; custom roles are allowed.",
+      "items": {
+        "type": "object",
+        "required": ["source", "role"],
+        "properties": {
+          "source": {"type": "string", "minLength": 1},
+          "role": {"type": "string", "minLength": 1},
+          "n_rows": {"type": "integer", "minimum": 0},
+          "notes": {"type": "string"},
+          "metadata": {"type": "object"}
+        },
+        "additionalProperties": true
+      }
+    },
+    "guardrails": {
+      "type": "array",
+      "description": "Optional predeclared guardrails for claim/evidence discipline.",
+      "items": {"type": "string", "minLength": 1}
+    },
+    "prediction_artifacts": {
+      "type": "array",
+      "description": "Optional references to retained prediction artifacts.",
+      "items": {
+        "type": "object",
+        "required": ["uri", "media_type", "columns"],
+        "properties": {
+          "uri": {"type": "string", "minLength": 1},
+          "media_type": {"type": "string", "minLength": 1},
+          "sha256": {"type": "string"},
+          "n_rows": {"type": "integer", "minimum": 0},
+          "role": {
+            "oneOf": [
+              {"type": "string", "minLength": 1},
+              {
+                "type": "array",
+                "minItems": 1,
+                "items": {"type": "string", "minLength": 1}
+              }
+            ]
+          },
+          "metadata": {"type": "object"},
+          "columns": {
+            "type": "object",
+            "required": ["label", "score"],
+            "properties": {
+              "row_id": {"type": "string", "minLength": 1},
+              "content_hash": {"type": "string", "minLength": 1},
+              "label": {"type": "string", "minLength": 1},
+              "score": {"type": "string", "minLength": 1},
+              "scorer": {"type": "string", "minLength": 1},
+              "slice": {"type": "string", "minLength": 1},
+              "text": {"type": "string", "minLength": 1},
+              "provenance": {
+                "type": "object",
+                "additionalProperties": {"type": "string"}
+              }
+            },
+            "additionalProperties": true
+          }
+        },
+        "additionalProperties": true
+      }
+    }
+  },
+  "additionalProperties": true
+}

eval_toolkit/schemas/manifest.v3.json

@@ -0,0 +1,186 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://eval-toolkit/schemas/manifest.v3.json",
+  "version": "3",
+  "title": "eval-toolkit RunManifest (v3)",
+  "description": "Schema for run_dir/manifest.json. Produced by eval_toolkit.manifest.write_manifest. NeurIPS Reproducibility Checklist-aligned. v3 adds: contamination_flags (required object; per-scorer contamination posture audit trail). v2 fields preserved. Default schema for new manifests starting v0.23.0.",
+  "type": "object",
+  "required": ["schema_version", "run_id", "captured_at", "code_versions", "env", "contamination_flags"],
+  "properties": {
+    "schema_version": {
+      "const": "v3"
+    },
+    "run_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "captured_at": {
+      "type": "string",
+      "description": "ISO-8601 UTC timestamp captured at build_manifest() call time.",
+      "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+    },
+    "git_sha": {
+      "type": ["string", "null"]
+    },
+    "dirty_flag": {
+      "type": "boolean"
+    },
+    "code_versions": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "data_revisions": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "seeds": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "integer"
+      }
+    },
+    "data_hashes": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "config_hash": {
+      "type": "string"
+    },
+    "env": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "gpu_info": {
+      "type": "object",
+      "properties": {
+        "name": {"type": "string"},
+        "count": {"type": "integer", "minimum": 0},
+        "memory_gb": {"type": "number", "minimum": 0}
+      },
+      "additionalProperties": true
+    },
+    "cuda_version": {
+      "type": ["string", "null"]
+    },
+    "wall_clock_seconds": {
+      "type": ["number", "null"]
+    },
+    "versioned_objects": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "leakage_report": {
+      "type": ["object", "null"],
+      "properties": {
+        "findings": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["check_name", "severity", "drop_indices", "evidence", "message", "n_affected"],
+            "properties": {
+              "check_name": {"type": "string"},
+              "severity": {"enum": ["error", "warning", "info"]},
+              "drop_indices": {"type": ["object", "null"]},
+              "evidence": {"type": "object"},
+              "message": {"type": "string"},
+              "n_affected": {"type": "integer", "minimum": 0}
+            }
+          }
+        }
+      }
+    },
+    "source_roles": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["source", "role"],
+        "properties": {
+          "source": {"type": "string", "minLength": 1},
+          "role": {"type": "string", "minLength": 1},
+          "n_rows": {"type": "integer", "minimum": 0},
+          "notes": {"type": "string"},
+          "metadata": {"type": "object"}
+        },
+        "additionalProperties": true
+      }
+    },
+    "guardrails": {
+      "type": "array",
+      "description": "Optional predeclared guardrails for claim/evidence discipline. v3 may use either string entries (v2 back-compat) or structured objects (e.g., source_freshness_check sub-objects with timestamp + method).",
+      "items": {
+        "oneOf": [
+          {"type": "string", "minLength": 1},
+          {"type": "object", "minProperties": 1}
+        ]
+      }
+    },
+    "contamination_flags": {
+      "type": "object",
+      "description": "v3 addition. Per-scorer contamination posture flags. Keys are scorer names; values are enum strings. Required field; may be an empty object for runs not tracking contamination (e.g., eval-toolkit's own test runs).",
+      "additionalProperties": {
+        "type": "string",
+        "enum": ["verified_disjoint", "suspected_contamination", "vendor_black_box", "unknown"]
+      }
+    },
+    "prediction_artifacts": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["uri", "media_type", "columns"],
+        "properties": {
+          "uri": {"type": "string", "minLength": 1},
+          "media_type": {"type": "string", "minLength": 1},
+          "sha256": {"type": "string"},
+          "n_rows": {"type": "integer", "minimum": 0},
+          "role": {
+            "oneOf": [
+              {"type": "string", "minLength": 1},
+              {
+                "type": "array",
+                "minItems": 1,
+                "items": {"type": "string", "minLength": 1}
+              }
+            ]
+          },
+          "metadata": {"type": "object"},
+          "columns": {
+            "type": "object",
+            "required": ["label", "score"],
+            "properties": {
+              "row_id": {"type": "string", "minLength": 1},
+              "content_hash": {"type": "string", "minLength": 1},
+              "label": {"type": "string", "minLength": 1},
+              "score": {"type": "string", "minLength": 1},
+              "scorer": {"type": "string", "minLength": 1},
+              "slice": {"type": "string", "minLength": 1},
+              "text": {"type": "string", "minLength": 1},
+              "provenance": {
+                "type": "object",
+                "additionalProperties": {"type": "string"}
+              }
+            },
+            "additionalProperties": true
+          }
+        },
+        "additionalProperties": true
+      }
+    }
+  },
+  "additionalProperties": true
+}

eval_toolkit/schemas/results.v1.json

@@ -0,0 +1,87 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://eval-toolkit/schemas/results.v1.json",
+  "version": "1",
+  "title": "eval-toolkit RunResult (compact)",
+  "description": "Schema for run_dir/results.json (per-row scores stripped). Produced by eval_toolkit.harness.write_run_result.",
+  "type": "object",
+  "required": ["schema_version", "run_id", "config", "by_slice"],
+  "properties": {
+    "schema_version": {
+      "const": "v1",
+      "description": "Stable schema version. Bumps on breaking output changes."
+    },
+    "run_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "git_sha": {
+      "type": ["string", "null"]
+    },
+    "config": {
+      "type": "object",
+      "description": "Eval-time config (n_resamples, seed, scorers, slices, paired_diffs, on_scorer_error, optional leakage_report)."
+    },
+    "by_slice": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "description": "Per-slice block: n, n_positive, by_scorer, paired_diffs. Scorer blocks may optionally include transferred_operating_points for validation-fit thresholds applied to this slice."
+      }
+    },
+    "by_fold": {
+      "type": "object",
+      "description": "Per-fold raw RunResults (populated by evaluate_folded). Empty for non-folded runs.",
+      "additionalProperties": true
+    },
+    "fold_summary": {
+      "type": "object",
+      "description": "CV-CI summary [slice][scorer][metric] = {mean, ci_low, ci_high, n_folds}. Empty for non-folded runs.",
+      "additionalProperties": true
+    },
+    "claim_report": {
+      "type": "object",
+      "description": "Optional ClaimReport payload from eval_toolkit.claims.evaluate_claims.",
+      "additionalProperties": true
+    },
+    "prediction_artifacts": {
+      "type": "array",
+      "description": "Optional references to retained prediction artifacts associated with the result.",
+      "items": {"type": "object", "additionalProperties": true}
+    },
+    "evidence_axes": {
+      "type": "array",
+      "description": "Optional named axes that scope aggregate evidence.",
+      "items": {
+        "type": "object",
+        "required": ["name", "value"],
+        "properties": {
+          "name": {"type": "string", "minLength": 1},
+          "value": {"type": "string", "minLength": 1}
+        },
+        "additionalProperties": true
+      }
+    },
+    "pairing_metadata": {
+      "type": "object",
+      "description": "Optional explicit pairing metadata for paired comparisons.",
+      "additionalProperties": true
+    },
+    "aggregate_evidence": {
+      "type": "object",
+      "description": "Optional typed aggregate status: inferential, descriptive, diagnostic, or unsupported.",
+      "properties": {
+        "status": {
+          "enum": ["inferential", "descriptive", "diagnostic", "unsupported"]
+        }
+      },
+      "additionalProperties": true
+    },
+    "threshold_policy": {
+      "type": "object",
+      "description": "Optional threshold policy metadata naming calibration slice, score column, selector, constraints, and claim-enabled status.",
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": true
+}

eval_toolkit/schemas/results_full.v1.json

@@ -0,0 +1,83 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://eval-toolkit/schemas/results_full.v1.json",
+  "version": "1",
+  "title": "eval-toolkit RunResult (full)",
+  "description": "Schema for run_dir/results_full.json (includes per-row scores). Same top-level shape as results.v1; differs only in that by_slice[*].by_scorer[*].scores arrays are retained.",
+  "type": "object",
+  "required": ["schema_version", "run_id", "config", "by_slice"],
+  "properties": {
+    "schema_version": {
+      "const": "v1"
+    },
+    "run_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "git_sha": {
+      "type": ["string", "null"]
+    },
+    "config": {
+      "type": "object"
+    },
+    "by_slice": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "description": "Per-slice block. Scorer blocks may optionally include transferred_operating_points for validation-fit thresholds applied to this slice."
+      }
+    },
+    "by_fold": {
+      "type": "object",
+      "additionalProperties": true
+    },
+    "fold_summary": {
+      "type": "object",
+      "additionalProperties": true
+    },
+    "claim_report": {
+      "type": "object",
+      "description": "Optional ClaimReport payload from eval_toolkit.claims.evaluate_claims.",
+      "additionalProperties": true
+    },
+    "prediction_artifacts": {
+      "type": "array",
+      "description": "Optional references to retained prediction artifacts associated with the result.",
+      "items": {"type": "object", "additionalProperties": true}
+    },
+    "evidence_axes": {
+      "type": "array",
+      "description": "Optional named axes that scope aggregate evidence.",
+      "items": {
+        "type": "object",
+        "required": ["name", "value"],
+        "properties": {
+          "name": {"type": "string", "minLength": 1},
+          "value": {"type": "string", "minLength": 1}
+        },
+        "additionalProperties": true
+      }
+    },
+    "pairing_metadata": {
+      "type": "object",
+      "description": "Optional explicit pairing metadata for paired comparisons.",
+      "additionalProperties": true
+    },
+    "aggregate_evidence": {
+      "type": "object",
+      "description": "Optional typed aggregate status: inferential, descriptive, diagnostic, or unsupported.",
+      "properties": {
+        "status": {
+          "enum": ["inferential", "descriptive", "diagnostic", "unsupported"]
+        }
+      },
+      "additionalProperties": true
+    },
+    "threshold_policy": {
+      "type": "object",
+      "description": "Optional threshold policy metadata naming calibration slice, score column, selector, constraints, and claim-enabled status.",
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": true
+}

eval_toolkit/seeds.py

@@ -0,0 +1,119 @@
+"""Global seed-setting for reproducibility.
+
+Public API: :func:`set_global_seeds`. Sets ``PYTHONHASHSEED``, ``random``,
+``numpy``, and optionally ``torch`` (with ``cudnn.deterministic=True``).
+Torch is a soft dependency — toolkit installs without it.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import warnings
+
+import numpy as np
+
+__all__ = ["set_global_seeds"]
+
+
+def set_global_seeds(seed: int, *, strict_torch_determinism: bool = False) -> None:
+    """Pin random / numpy / torch seeds + enable deterministic CUDA where possible.
+
+    Sets the four sources of nondeterminism that any Python-flavored ML
+    pipeline can practically control: ``PYTHONHASHSEED`` (string-hash
+    salt), the stdlib ``random`` module, ``numpy``'s legacy global RNG,
+    and ``torch`` (CPU + CUDA + cuDNN flags) when torch is importable.
+
+    Matches PyTorch Lightning's :func:`seed_everything` discipline.
+
+    Parameters
+    ----------
+    seed : int
+        Non-negative seed value.
+    strict_torch_determinism : bool, optional
+        If ``True``, also call ``torch.use_deterministic_algorithms(True)``
+        to make every torch op raise on nondeterministic kernels (vs. the
+        default cuDNN-only flags). Useful when bit-exact reproducibility
+        is required across runs. Has no effect if torch is not installed.
+        Default ``False``.
+
+    Raises
+    ------
+    TypeError
+        If ``seed`` is not an ``int``.
+    ValueError
+        If ``seed`` is negative.
+    RuntimeError
+        If ``strict_torch_determinism=True`` is requested but ``torch`` is
+        not installed.
+
+    Notes
+    -----
+    Full determinism on CUDA is not always achievable — some kernels lack
+    deterministic implementations. Documented residual variance can be
+    handled by reporting multi-seed CIs rather than relying on bit-exact
+    reproducibility.
+
+    ``PYTHONHASHSEED`` is set in the *current* process's environment but
+    only takes effect for child processes (the current interpreter has
+    already initialized its hash randomization). To get deterministic
+    string hashing in the current run, set ``PYTHONHASHSEED`` from the
+    shell *before* launching Python; this function emits a UserWarning
+    when the variable is already set to a different value.
+
+    Torch is imported lazily inside the function. If ``torch`` is not
+    installed, the random + numpy seeds + ``PYTHONHASHSEED`` are still
+    set.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> set_global_seeds(42)
+    >>> a1 = np.random.rand(5)
+    >>> set_global_seeds(42)
+    >>> a2 = np.random.rand(5)
+    >>> bool(np.all(a1 == a2))
+    True
+
+    See Also
+    --------
+    `pytorch-lightning.seed_everything <https://lightning.ai/docs/pytorch/stable/common/seed.html>`_ :
+        Reference for the strict_torch_determinism semantics.
+    """
+    if not isinstance(seed, int):
+        raise TypeError(f"seed must be int, got {type(seed).__name__}")
+    if seed < 0:
+        raise ValueError(f"seed must be non-negative, got {seed}")
+
+    target = str(seed)
+    existing = os.environ.get("PYTHONHASHSEED")
+    if existing is not None and existing != target:
+        warnings.warn(
+            f"PYTHONHASHSEED was already set to {existing!r}; overwriting to "
+            f"{target!r}. The current Python process's hash randomization is "
+            f"unaffected — set PYTHONHASHSEED before launching Python for "
+            f"in-process determinism.",
+            UserWarning,
+            stacklevel=2,
+        )
+    os.environ["PYTHONHASHSEED"] = target
+
+    random.seed(seed)
+    np.random.seed(seed)
+
+    try:
+        import torch  # type: ignore[import-not-found]  # noqa: PLC0415
+    except ImportError:
+        if strict_torch_determinism:
+            raise RuntimeError(
+                "strict_torch_determinism=True requested but torch is not installed"
+            ) from None
+        return
+
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    if strict_torch_determinism:
+        torch.use_deterministic_algorithms(True)