invarlock 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

invarlock/reporting/{certificate_schema.py → report_schema.py}

@@ -11,16 +11,16 @@ except Exception:  # pragma: no cover
     jsonschema = None
 
 
-# Certificate schema version (PM-first canonical)
-CERTIFICATE_SCHEMA_VERSION = "v1"
+# Evaluation report schema version (PM-first canonical)
+REPORT_SCHEMA_VERSION = "v1"
 
 
-# Minimal JSON Schema describing the canonical shape of a certificate.
+# Minimal JSON Schema describing the canonical shape of an evaluation report.
 # This focuses on structural validity; numerical thresholds are validated
 # separately in metric-specific logic.
-CERTIFICATE_JSON_SCHEMA: dict[str, Any] = {
+REPORT_JSON_SCHEMA: dict[str, Any] = {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "title": "InvarLock Safety Certificate",
+    "title": "InvarLock Evaluation Report",
     "type": "object",
     "required": [
         "schema_version",
@@ -32,7 +32,7 @@ CERTIFICATE_JSON_SCHEMA: dict[str, Any] = {
         "primary_metric",
     ],
     "properties": {
-        "schema_version": {"const": CERTIFICATE_SCHEMA_VERSION},
+        "schema_version": {"const": REPORT_SCHEMA_VERSION},
         "run_id": {"type": "string", "minLength": 4},
         "edit_name": {"type": "string"},
         "policy_digest": {
@@ -179,21 +179,21 @@ def _load_validation_allowlist() -> set[str]:
     return set(_VALIDATION_ALLOWLIST_DEFAULT)
 
 
-def _validate_with_jsonschema(certificate: dict[str, Any]) -> bool:
-    """Validate certificate with JSON Schema when available."""
+def _validate_with_jsonschema(report: dict[str, Any]) -> bool:
+    """Validate evaluation report with JSON Schema when available."""
     if jsonschema is None:
         return True  # Schema library unavailable; fall back to minimal checks
     try:
-        jsonschema.validate(instance=certificate, schema=CERTIFICATE_JSON_SCHEMA)
+        jsonschema.validate(instance=report, schema=REPORT_JSON_SCHEMA)
         return True
     except Exception:
         return False
 
 
-def validate_certificate(certificate: dict[str, Any]) -> bool:
-    """Validate certificate structure and essential flags."""
+def validate_report(report: dict[str, Any]) -> bool:
+    """Validate evaluation report structure and essential flags."""
     try:
-        if certificate.get("schema_version") != CERTIFICATE_SCHEMA_VERSION:
+        if report.get("schema_version") != REPORT_SCHEMA_VERSION:
             return False
 
         # Prefer JSON Schema structural validation; if unavailable or too strict,
@@ -202,20 +202,20 @@ def validate_certificate(certificate: dict[str, Any]) -> bool:
         # disallow unknown validation keys at schema level.
         try:
             vkeys = _load_validation_allowlist()
-            if isinstance(CERTIFICATE_JSON_SCHEMA.get("properties"), dict):
-                vspec = CERTIFICATE_JSON_SCHEMA["properties"].get("validation")
+            if isinstance(REPORT_JSON_SCHEMA.get("properties"), dict):
+                vspec = REPORT_JSON_SCHEMA["properties"].get("validation")
                 if isinstance(vspec, dict):
                     vspec["properties"] = {k: {"type": "boolean"} for k in vkeys}
                     vspec["additionalProperties"] = False
         except Exception:
             pass
 
-        if not _validate_with_jsonschema(certificate):
+        if not _validate_with_jsonschema(report):
             # Minimal fallback: require schema version + run_id + primary_metric
-            run_id_ok = isinstance(certificate.get("run_id"), str) and bool(
-                certificate.get("run_id")
+            run_id_ok = isinstance(report.get("run_id"), str) and bool(
+                report.get("run_id")
             )
-            pm = certificate.get("primary_metric")
+            pm = report.get("primary_metric")
             pm_ok = isinstance(pm, dict) and (
                 isinstance(pm.get("final"), int | float)
                 or (isinstance(pm.get("kind"), str) and bool(pm.get("kind")))
@@ -223,7 +223,7 @@ def validate_certificate(certificate: dict[str, Any]) -> bool:
             if not (run_id_ok and pm_ok):
                 return False
 
-        validation = certificate.get("validation", {})
+        validation = report.get("validation", {})
         for flag in [
             "preview_final_drift_acceptable",
             "primary_metric_acceptable",
@@ -242,7 +242,7 @@ def validate_certificate(certificate: dict[str, Any]) -> bool:
 
 
 __all__ = [
-    "CERTIFICATE_SCHEMA_VERSION",
-    "CERTIFICATE_JSON_SCHEMA",
-    "validate_certificate",
+    "REPORT_SCHEMA_VERSION",
+    "REPORT_JSON_SCHEMA",
+    "validate_report",
 ]

invarlock/reporting/report_types.py

@@ -34,7 +34,7 @@ class MetaData(TypedDict):
     """Metadata about the model and execution environment."""
 
     model_id: str  # Model identifier (e.g., "gpt2", "path/to/model")
-    adapter: str  # Adapter name (e.g., "hf_gpt2")
+    adapter: str  # Adapter name (e.g., "hf_causal")
     commit: str  # Git commit SHA
     seed: int  # Random seed used for evaluation
     device: str  # Device used ("cpu", "cuda", "mps")
@@ -107,6 +107,11 @@ class EvalMetrics(TypedDict, total=False):
     # Optional aux fields retained for guard telemetry and debug
     latency_ms_per_tok: float  # Average latency per token in milliseconds
     memory_mb_peak: float  # Peak memory usage in MB
+    gpu_memory_mb_peak: float  # Peak GPU memory usage in MB
+    gpu_memory_reserved_mb_peak: float  # Peak GPU reserved memory in MB
+    timings: dict[str, float]  # Phase timing breakdown (seconds)
+    guard_timings: dict[str, float]  # Per-guard timings (seconds)
+    memory_snapshots: list[dict[str, Any]]  # Phase memory snapshots
     spectral: dict[str, Any]  # Spectral norm summaries
     rmt: dict[str, Any]  # RMT statistics
     invariants: dict[str, Any]  # Model invariant check results

invarlock/reporting/telemetry.py

@@ -0,0 +1,86 @@
+"""
+Telemetry report utilities.
+
+Produces a compact JSON summary for performance analysis.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+
+def build_telemetry_payload(report: dict[str, Any]) -> dict[str, Any]:
+    """Build a structured telemetry payload from a run report."""
+    meta_in = report.get("meta", {}) if isinstance(report, dict) else {}
+    metrics_in = report.get("metrics", {}) if isinstance(report, dict) else {}
+
+    payload: dict[str, Any] = {"generated_at": datetime.now().isoformat()}
+
+    if isinstance(meta_in, dict):
+        payload["meta"] = {
+            "model_id": meta_in.get("model_id"),
+            "adapter": meta_in.get("adapter"),
+            "device": meta_in.get("device"),
+            "run_id": meta_in.get("run_id"),
+            "profile": meta_in.get("profile"),
+        }
+
+    if isinstance(metrics_in, dict):
+        timings = metrics_in.get("timings")
+        if isinstance(timings, dict):
+            payload["timings"] = timings
+
+        guard_timings = metrics_in.get("guard_timings")
+        if isinstance(guard_timings, dict):
+            payload["guard_timings"] = guard_timings
+
+        memory_snapshots = metrics_in.get("memory_snapshots")
+        if isinstance(memory_snapshots, list):
+            payload["memory_snapshots"] = memory_snapshots
+
+        memory_summary: dict[str, Any] = {}
+        for key in (
+            "memory_mb_peak",
+            "gpu_memory_mb_peak",
+            "gpu_memory_reserved_mb_peak",
+        ):
+            value = metrics_in.get(key)
+            if isinstance(value, int | float):
+                memory_summary[key] = float(value)
+        if memory_summary:
+            payload["memory"] = memory_summary
+
+        perf_metrics: dict[str, Any] = {}
+        for key in (
+            "latency_ms_per_tok",
+            "throughput_tok_per_s",
+            "eval_samples",
+            "total_tokens",
+        ):
+            value = metrics_in.get(key)
+            if isinstance(value, int | float):
+                perf_metrics[key] = float(value)
+        if perf_metrics:
+            payload["performance"] = perf_metrics
+
+    return payload
+
+
+def save_telemetry_report(
+    report: dict[str, Any],
+    output_dir: Path,
+    *,
+    filename: str = "telemetry.json",
+) -> Path:
+    """Write telemetry JSON payload to the output directory."""
+    payload = build_telemetry_payload(report)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = output_dir / filename
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
+    return path
+
+
+__all__ = ["build_telemetry_payload", "save_telemetry_report"]

invarlock-0.3.8.dist-info/METADATA

@@ -0,0 +1,283 @@
+Metadata-Version: 2.4
+Name: invarlock
+Version: 0.3.8
+Summary: Edit‑agnostic robustness evaluation reports for weight edits (InvarLock framework)
+Author-email: InvarLock Team <oss@invarlock.dev>
+Maintainer-email: InvarLock Maintainers <support@invarlock.dev>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/invarlock/invarlock
+Project-URL: Repository, https://github.com/invarlock/invarlock
+Project-URL: Documentation, https://github.com/invarlock/invarlock/tree/main/docs
+Project-URL: Issues, https://github.com/invarlock/invarlock/issues
+Project-URL: Changelog, https://github.com/invarlock/invarlock/blob/main/CHANGELOG.md
+Keywords: machine-learning,deep-learning,transformers,pytorch,llm,quantization,safety,evaluation,certification
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.15
+Requires-Dist: click>=8.1
+Requires-Dist: shellingham>=1.5.0
+Requires-Dist: pandas>=2.2
+Requires-Dist: scikit-learn>=1.4
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: markdown>=3.5
+Requires-Dist: psutil>=5.9
+Requires-Dist: hypothesis>=6.98
+Requires-Dist: typing_extensions>=4.7
+Requires-Dist: jsonschema>=4.0
+Provides-Extra: adapters
+Requires-Dist: torch>=2.1.0; extra == "adapters"
+Requires-Dist: transformers>=5.0.0; extra == "adapters"
+Provides-Extra: hf
+Requires-Dist: torch>=2.1.0; extra == "hf"
+Requires-Dist: transformers>=5.0.0; extra == "hf"
+Requires-Dist: datasets>=3.0; extra == "hf"
+Requires-Dist: numpy>=1.24; extra == "hf"
+Requires-Dist: huggingface_hub>=1.0.0; extra == "hf"
+Requires-Dist: aiohttp>=3.12.14; extra == "hf"
+Requires-Dist: h2>=4.3.0; extra == "hf"
+Requires-Dist: pillow>=11.3.0; extra == "hf"
+Provides-Extra: guards
+Requires-Dist: torch>=2.1.0; extra == "guards"
+Requires-Dist: numpy>=1.24; extra == "guards"
+Provides-Extra: edits
+Requires-Dist: torch>=2.1.0; extra == "edits"
+Provides-Extra: eval
+Requires-Dist: torch>=2.1.0; extra == "eval"
+Requires-Dist: datasets>=3.0; extra == "eval"
+Provides-Extra: gptq
+Requires-Dist: torch>=2.1.0; extra == "gptq"
+Requires-Dist: auto-gptq>=0.7.0; platform_system == "Linux" and extra == "gptq"
+Requires-Dist: triton>=2.3.0; platform_system == "Linux" and extra == "gptq"
+Requires-Dist: transformers>=5.0.0; extra == "gptq"
+Provides-Extra: awq
+Requires-Dist: torch>=2.1.0; extra == "awq"
+Requires-Dist: autoawq>=0.2.0; platform_system == "Linux" and extra == "awq"
+Requires-Dist: transformers>=5.0.0; extra == "awq"
+Requires-Dist: triton>=2.3.0; platform_system == "Linux" and extra == "awq"
+Provides-Extra: gpu
+Requires-Dist: torch>=2.1.0; extra == "gpu"
+Requires-Dist: accelerate>=0.27; extra == "gpu"
+Requires-Dist: bitsandbytes>=0.41; platform_system == "Linux" and extra == "gpu"
+Provides-Extra: all
+Requires-Dist: torch>=2.1.0; extra == "all"
+Requires-Dist: transformers>=5.0.0; extra == "all"
+Requires-Dist: datasets>=3.0; extra == "all"
+Requires-Dist: numpy>=1.24; extra == "all"
+Requires-Dist: huggingface_hub>=1.0.0; extra == "all"
+Requires-Dist: accelerate>=0.27; extra == "all"
+Requires-Dist: bitsandbytes>=0.41; platform_system == "Linux" and extra == "all"
+Requires-Dist: auto-gptq>=0.7.0; platform_system == "Linux" and extra == "all"
+Requires-Dist: autoawq>=0.2.0; platform_system == "Linux" and extra == "all"
+Requires-Dist: triton>=2.3.0; platform_system == "Linux" and extra == "all"
+Requires-Dist: aiohttp>=3.12.14; extra == "all"
+Requires-Dist: h2>=4.3.0; extra == "all"
+Requires-Dist: pillow>=11.3.0; extra == "all"
+Provides-Extra: onnx
+Requires-Dist: optimum>=1.17.0; extra == "onnx"
+Requires-Dist: onnxruntime>=1.17.0; extra == "onnx"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: black>=23.0; extra == "dev"
+Requires-Dist: mypy>=1.0; extra == "dev"
+Requires-Dist: hypothesis>=6.98; extra == "dev"
+Requires-Dist: pre-commit>=3.0; extra == "dev"
+Requires-Dist: mkdocs>=1.5; extra == "dev"
+Requires-Dist: mkdocs-material>=9.5; extra == "dev"
+Requires-Dist: mkdocs-mermaid2-plugin>=1.1; extra == "dev"
+Requires-Dist: sphinx>=7.0; extra == "dev"
+Requires-Dist: matplotlib>=3.7; extra == "dev"
+Requires-Dist: bitsandbytes>=0.41; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img
+    src="docs/assets/invarlock-logo.svg"
+    alt="InvarLock"
+    width="420"
+  />
+</p>
+
+# InvarLock — Edit‑agnostic robustness reports for weight edits
+
+[![CI](https://img.shields.io/github/actions/workflow/status/invarlock/invarlock/ci.yml?branch=main&logo=github&label=CI)](https://github.com/invarlock/invarlock/actions/workflows/ci.yml)
+[![PyPI](https://badge.fury.io/py/invarlock.svg)](https://pypi.org/project/invarlock/)
+[![Docs](https://img.shields.io/badge/docs-quickstart-blue.svg)](https://github.com/invarlock/invarlock/blob/main/docs/user-guide/quickstart.md)
+[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/release/python-3120/)
+
+Quantizing, pruning, or otherwise editing a model’s weights can silently degrade quality.
+InvarLock compares an edited **subject** checkpoint against a fixed **baseline** with paired
+evaluation windows, enforces a guard pipeline (invariants → spectral → RMT → variance), and
+produces a machine‑readable Evaluation Report you can gate in CI.
+
+> Status: pre‑1.0. Until 1.0, minor releases may be breaking. See `CHANGELOG.md`.
+
+For guidance on where to ask questions, how to report bugs, and what to expect in terms of response times, see
+[SUPPORT.md](https://github.com/invarlock/invarlock/blob/main/SUPPORT.md).
+
+## Why InvarLock?
+
+- **Quality gates for weight edits**: catch regressions before deployment.
+- **Statistical guarantees**: paired primary metrics with confidence intervals.
+- **Auditable evidence**: deterministic pairing metadata + policy digests in `evaluation.report.json`.
+- **CI/CD-friendly**: stable exit codes, `--json` outputs, and portable “proof packs”.
+- **Offline-first**: network is disabled by default; enable downloads per command.
+
+## Who is this for?
+
+- ML engineers shipping quantized/pruned checkpoints.
+- MLOps teams building CI quality gates and reviewable artifacts.
+- Researchers validating compression/edit methods with reproducible, paired eval.
+
+## How it works
+
+```text
+┌───────────────────────┐     ┌────────────────────────────────────────────┐
+│ Baseline (checkpoint) │────►│                                            │
+└───────────────────────┘     │  invarlock evaluate                        │
+                              │  ├─► Paired windows (deterministic)        │
+┌───────────────────────┐     │  ├─► GuardChain pipeline                   │
+│ Subject  (checkpoint) │────►│  │   └─► invariants → spectral → RMT → VE  │
+└───────────────────────┘     │  └─► Emit: evaluation.report.json          │    
+                              │                                            │
+                              └────────────────────────────────────────────┘                                                                                               
+                                                     │                                                                                                                          
+                                     ┌───────────────┴───────────────┐                                                                                                          
+                                     ▼                               ▼                                                                                                          
+                                 ✅ PASS                          ❌ FAIL                                                                                                        
+                                 (ship)                          (rollback)    
+                                     
+```
+
+## Quick start
+
+Colab (CPU-friendly):
+[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/invarlock/invarlock/blob/main/notebooks/invarlock_quickstart_cpu.ipynb)
+
+```bash
+# HF adapter stack (torch/transformers)
+pip install "invarlock[hf]"
+
+# Version + report schema (when available)
+invarlock --version
+
+# Compare baseline vs subject (downloads require explicit network enable)
+INVARLOCK_ALLOW_NETWORK=1 invarlock evaluate \
+  --baseline gpt2 \
+  --subject  gpt2 \
+  --adapter auto \
+  --profile dev \
+  --quiet
+
+# Validate the evaluation report
+invarlock verify reports/eval/evaluation.report.json
+
+# Render HTML for sharing
+invarlock report html -i reports/eval/evaluation.report.json -o reports/eval/evaluation.html
+```
+
+Example output (abridged; counts vary by profile/config):
+
+```text
+INVARLOCK v<version> · EVALUATE
+Baseline: gpt2 -> Subject: gpt2 · Profile: dev
+Status: PASS · Gates: <passed>/<total> passed
+Primary metric ratio: <ratio>
+Output: reports/eval/evaluation.report.json
+```
+
+## Proof packs (portable evidence bundles)
+
+Proof packs bundle reports + verification metadata into a distributable artifact.
+
+- Guide: <https://github.com/invarlock/invarlock/blob/main/docs/user-guide/proof-packs.md>
+- Verify: `scripts/proof_packs/verify_pack.sh --pack <dir> --strict` (or `PACK_STRICT_MODE=1 ...`)
+
+Note: `configs/` and `scripts/` are repo resources and are not shipped in wheels; clone the repo to use
+presets and proof-pack helpers.
+
+## Installation
+
+```bash
+# Minimal CLI (no torch/transformers)
+pip install invarlock
+
+# HF workflows (torch/transformers)
+pip install "invarlock[hf]"
+```
+
+Optional extras: `invarlock[gpu]`, `invarlock[awq,gptq]`. Full setup: <https://github.com/invarlock/invarlock/blob/main/docs/user-guide/getting-started.md>.
+
+## Documentation
+
+- Quickstart: <https://github.com/invarlock/invarlock/blob/main/docs/user-guide/quickstart.md>
+- Compare & evaluate (BYOE): <https://github.com/invarlock/invarlock/blob/main/docs/user-guide/compare-and-evaluate.md>
+- Reading a report: <https://github.com/invarlock/invarlock/blob/main/docs/user-guide/reading-report.md>
+- CLI reference: <https://github.com/invarlock/invarlock/blob/main/docs/reference/cli.md>
+- Assurance case: <https://github.com/invarlock/invarlock/blob/main/docs/assurance/00-safety-case.md>
+- Threat model: <https://github.com/invarlock/invarlock/blob/main/docs/security/threat-model.md>
+
+## Community
+
+- Questions/ideas: <https://github.com/invarlock/invarlock/discussions>
+- Bug reports: <https://github.com/invarlock/invarlock/issues>
+- Contact: <mailto:support@invarlock.dev>
+
+## Citation
+
+If you use InvarLock in scientific work, please cite it (canonical metadata is in `CITATION.cff`):
+
+```bibtex
+@software{invarlock,
+  title  = {InvarLock: Edit-agnostic robustness evaluation reports for weight edits},
+  author = {{InvarLock Maintainers}},
+  url    = {https://github.com/invarlock/invarlock},
+}
+```
+
+## Limitations
+
+- InvarLock evaluates an edited model relative to a baseline under a specific configuration; results are not “global” guarantees.
+- Not a content-safety/alignment tool.
+- Native Windows is not supported (use WSL2 or Linux).
+
+## Support matrix
+
+<!-- markdownlint-disable MD060 -->
+| Platform               | Status          | Notes                                     |
+| ---------------------- | --------------- | ----------------------------------------- |
+| Python 3.12+           | ✅ Required      |                                           |
+| Linux                  | ✅ Full          | Primary dev target                        |
+| macOS (Intel/M-series) | ✅ Full          | MPS supported (default on Apple Silicon)  |
+| Windows                | ❌ Not supported | Use WSL2 or a Linux container if required |
+| CUDA                   | ✅ Recommended   | For larger models                         |
+| CPU                    | ✅ Fallback      | Slower but functional                     |
+<!-- markdownlint-enable MD060 -->
+
+## Contributing
+
+- Contributing guide: <https://github.com/invarlock/invarlock/blob/main/CONTRIBUTING.md>
+- Fast local checks (repo clone):
+  - `make dev-install`
+  - `make test`
+  - `make lint`
+
+## License
+
+Apache-2.0 — see `LICENSE`.