system-design-primer-skill 1.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -60
- package/SKILL.md +437 -93
- package/VERSION +1 -1
- package/evals/README.md +1022 -0
- package/evals/system-design-cases.json +414 -0
- package/package.json +46 -4
- package/references/03-data-and-storage.md +8 -2
- package/references/04-caching.md +3 -3
- package/references/05-async-and-messaging.md +4 -0
- package/references/06-distributed-systems.md +2 -2
- package/references/07-reliability-and-resilience.md +7 -3
- package/references/08-api-security-observability.md +4 -0
- package/references/10-workflows-and-checklists.md +1 -1
- package/references/13-architecture-intelligence.md +91 -0
- package/references/14-api-and-data-contracts.md +80 -0
- package/references/15-failure-analysis.md +88 -0
- package/references/16-multi-agent-architecture.md +76 -0
- package/references/17-agent-harness-and-runtime.md +87 -0
- package/references/18-mcp-architecture.md +67 -0
- package/references/19-agent-security-and-permissions.md +90 -0
- package/references/20-agent-state-and-memory.md +76 -0
- package/references/21-agent-evaluation-and-observability.md +90 -0
- package/references/22-kubernetes-and-container-platforms.md +29 -0
- package/references/23-service-mesh-and-traffic-management.md +21 -0
- package/references/24-multi-region-and-disaster-recovery.md +28 -0
- package/references/25-platform-engineering-and-developer-experience.md +26 -0
- package/references/26-finops-and-cloud-cost.md +29 -0
- package/references/27-production-observability-and-slo.md +33 -0
- package/references/28-event-streaming-and-kafka-pulsar.md +33 -0
- package/references/29-event-sourcing-and-cqrs.md +23 -0
- package/references/30-change-data-capture-and-debezium.md +27 -0
- package/references/31-data-lakehouse-and-stream-batch.md +24 -0
- package/references/32-workflow-orchestration-and-temporal.md +28 -0
- package/references/33-schema-registry-and-event-governance.md +22 -0
- package/references/34-postgresql-internals-and-performance.md +45 -0
- package/references/35-mysql-innodb-and-performance.md +29 -0
- package/references/36-sharding-and-distributed-sql.md +22 -0
- package/references/37-consensus-and-distributed-transactions.md +29 -0
- package/references/38-search-architecture-elasticsearch-opensearch.md +24 -0
- package/references/39-vector-databases-and-hybrid-retrieval.md +31 -0
- package/references/40-zero-downtime-database-migrations.md +27 -0
- package/references/41-api-gateway-and-edge-architecture.md +23 -0
- package/references/42-rest-api-design-and-lifecycle.md +23 -0
- package/references/43-graphql-architecture-and-governance.md +16 -0
- package/references/44-grpc-and-service-contracts.md +16 -0
- package/references/45-websocket-sse-and-realtime-systems.md +16 -0
- package/references/46-realtime-collaboration-ot-crdt.md +16 -0
- package/references/47-api-security-rate-limiting-and-abuse.md +16 -0
- package/references/48-oauth-oidc-and-passkeys.md +20 -0
- package/references/49-iam-rbac-abac-rebac.md +20 -0
- package/references/50-zero-trust-and-workload-identity.md +16 -0
- package/references/51-secrets-kms-pki-and-key-management.md +20 -0
- package/references/52-supply-chain-security-sbom-slsa.md +78 -0
- package/references/53-threat-modeling-stride-and-attack-trees.md +16 -0
- package/references/54-privacy-engineering-and-data-governance.md +20 -0
- package/references/55-opentelemetry-and-instrumentation.md +54 -0
- package/references/56-distributed-tracing-and-context-propagation.md +47 -0
- package/references/57-continuous-profiling-and-performance-diagnostics.md +45 -0
- package/references/58-load-stress-soak-testing-and-workload-modeling.md +44 -0
- package/references/59-capacity-planning-and-performance-modeling.md +38 -0
- package/references/60-chaos-engineering-and-resilience-experiments.md +45 -0
- package/references/61-incident-management-and-postmortems.md +38 -0
- package/references/62-dns-and-service-discovery.md +35 -0
- package/references/63-load-balancing-and-global-traffic-management.md +31 -0
- package/references/64-cdn-and-edge-caching.md +34 -0
- package/references/65-cloud-networking-and-connectivity.md +34 -0
- package/references/66-edge-computing-architecture.md +33 -0
- package/references/67-storage-infrastructure-and-storage-systems.md +34 -0
- package/references/68-tcp-quic-and-network-performance.md +27 -0
- package/references/69-task-routing-and-evidence.md +115 -0
- package/scripts/audit_chain.py +36 -0
- package/scripts/compare_reports.py +43 -0
- package/scripts/evaluate.py +475 -0
- package/scripts/evaluation_catalog.py +129 -0
- package/scripts/evaluation_fingerprints.py +20 -0
- package/scripts/evidence_archive.py +79 -0
- package/scripts/evidence_signing.py +98 -0
- package/scripts/evidence_timestamp.py +441 -0
- package/scripts/install.ps1 +12 -3
- package/scripts/install.sh +15 -8
- package/scripts/report_integrity.py +184 -0
- package/scripts/review_consensus.py +77 -0
- package/scripts/review_evidence.py +105 -0
- package/scripts/review_provenance.py +44 -0
- package/scripts/reviewer_calibration.py +215 -0
- package/scripts/rubric_compatibility.py +134 -0
- package/scripts/run_evals.py +130 -0
- package/scripts/run_manifest.py +179 -0
- package/scripts/statistical_analysis.py +308 -0
- package/scripts/tsa_attestation.py +709 -0
- package/scripts/tsa_validation_report.py +627 -0
- package/scripts/validate.py +356 -11
- package/scripts/verify_build_provenance.py +625 -0
- package/scripts/verify_tsa_ocsp.py +962 -0
- package/scripts/verify_tsa_revocation.py +782 -0
- package/scripts/windows_broker_protocol.py +274 -0
- package/scripts/windows_broker_transport.py +141 -0
- package/scripts/windows_broker_win32.py +713 -0
- package/templates/agent-evaluation-plan-template.md +57 -0
- package/templates/agent-security-review-template.md +46 -0
- package/templates/agent-system-design-template.md +74 -0
- package/templates/api-architecture-design-template.md +33 -0
- package/templates/api-security-and-rate-limit-review-template.md +29 -0
- package/templates/architecture-decision-matrix-template.md +48 -0
- package/templates/architecture-review-scorecard-template.md +36 -0
- package/templates/cdc-data-pipeline-template.md +41 -0
- package/templates/cdn-and-edge-strategy-template.md +36 -0
- package/templates/chaos-experiment-template.md +39 -0
- package/templates/cloud-cost-model-template.md +37 -0
- package/templates/database-architecture-review-template.md +33 -0
- package/templates/edge-computing-readiness-template.md +36 -0
- package/templates/event-sourcing-cqrs-template.md +41 -0
- package/templates/event-streaming-design-template.md +47 -0
- package/templates/failure-analysis-template.md +36 -0
- package/templates/identity-and-access-architecture-template.md +29 -0
- package/templates/incident-postmortem-template.md +39 -0
- package/templates/multi-region-disaster-recovery-template.md +43 -0
- package/templates/network-architecture-design-template.md +38 -0
- package/templates/observability-architecture-template.md +52 -0
- package/templates/performance-test-plan-template.md +52 -0
- package/templates/privacy-data-governance-template.md +24 -0
- package/templates/production-readiness-checklist-template.md +52 -0
- package/templates/realtime-system-design-template.md +28 -0
- package/templates/security-threat-model-template.md +26 -0
- package/templates/sharding-distributed-sql-design-template.md +34 -0
- package/templates/storage-architecture-design-template.md +35 -0
- package/templates/supply-chain-security-review-template.md +20 -0
- package/templates/zero-downtime-database-migration-template.md +32 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Task routing and evidence discipline
|
|
2
|
+
|
|
3
|
+
## Purpose
|
|
4
|
+
|
|
5
|
+
Use this guide as the first-pass index for selecting knowledge. Route by the user's decision or observed failure, not by attempting to load the entire reference library. Start with one primary reference and add only the adjacent material needed to answer the concrete question. The table names the most relevant specialist references explicitly; do not make users or agents guess which file "matching" means.
|
|
6
|
+
|
|
7
|
+
## Cross-domain ownership rules
|
|
8
|
+
|
|
9
|
+
- **Data semantics:** `03-data-and-storage.md` owns portable data-model and transaction requirements; `34-postgresql-internals-and-performance.md` and `35-mysql-innodb-and-performance.md` own engine-specific behavior; `36-sharding-and-distributed-sql.md` owns partition movement and shard ownership.
|
|
10
|
+
- **API, identity, and observability:** `08-api-security-observability.md` is the overview; `47-api-security-rate-limiting-and-abuse.md` owns API abuse controls; `49-iam-rbac-abac-rebac.md` owns authorization policy semantics; `27-production-observability-and-slo.md` owns telemetry and SLO design.
|
|
11
|
+
- **Regional recovery:** `24-multi-region-and-disaster-recovery.md` owns regional replication, writer ownership, failover, and failback; `07-reliability-and-resilience.md` owns service-level resilience controls; `61-incident-management-and-postmortems.md` owns live incident command and postmortems.
|
|
12
|
+
- **Evidence boundary:** authentication, routing, telemetry, broker transactions, and workflow history are not substitutes for resource authorization, external-side-effect reconciliation, or proof of durable business completion. State the exact boundary and verification evidence.
|
|
13
|
+
|
|
14
|
+
## Security, privacy, and operations ownership
|
|
15
|
+
|
|
16
|
+
- **Agent authority:** `19-agent-security-and-permissions.md` owns model/tool threat controls; `49-iam-rbac-abac-rebac.md` owns authoritative authorization policy; `50-zero-trust-and-workload-identity.md` owns workload identity and peer authentication.
|
|
17
|
+
- **State and deletion:** `20-agent-state-and-memory.md` owns agent state categories and checkpoints; `54-privacy-engineering-and-data-governance.md` owns retention, subject rights, and deletion propagation across derived/replicated systems.
|
|
18
|
+
- **Cryptographic lifecycle:** `51-secrets-kms-pki-and-key-management.md` owns secret/key/certificate lifecycle; `67-storage-infrastructure-and-storage-systems.md` owns storage durability, recovery, and infrastructure behavior. Key rotation, backup retention, and deletion must be validated as separate operations.
|
|
19
|
+
- **Telemetry:** `55-opentelemetry-and-instrumentation.md` owns telemetry pipelines; `56-distributed-tracing-and-context-propagation.md` owns trace structure, propagation, baggage, sampling, and latency interpretation; `27-production-observability-and-slo.md` owns SLIs/SLOs, alerting, and production telemetry operations. `54-privacy-engineering-and-data-governance.md` owns authoritative data classification and retention policy. Missing telemetry does not prove missing work.
|
|
20
|
+
- **Consensus and transactions:** `37-consensus-and-distributed-transactions.md` owns consensus safety/liveness assumptions and cross-resource transaction guarantees; `06-distributed-systems.md` owns broad consistency/coordination concepts; `03-data-and-storage.md` owns database-local transaction semantics; `05-async-and-messaging.md` owns broker-agnostic delivery; `28-event-streaming-and-kafka-pulsar.md` owns broker-specific guarantees; `32-workflow-orchestration-and-temporal.md` owns durable process execution. Commit, acknowledgement, workflow history, and business completion are distinct.
|
|
21
|
+
- **Capacity evidence:** `59-capacity-planning-and-performance-modeling.md` owns conditional capacity models; `58-load-stress-soak-testing-and-workload-modeling.md` owns measured test methodology. State workload, topology, quota, and failure assumptions before claiming headroom.
|
|
22
|
+
|
|
23
|
+
## Task-to-reference routing
|
|
24
|
+
|
|
25
|
+
| Task signal | Start here | Add when relevant |
|
|
26
|
+
|---|---|---|
|
|
27
|
+
| Requirements, workload, sizing, latency or cost envelope | `01-requirements-and-capacity.md` | `59-capacity-planning-and-performance-modeling.md`, `58-load-stress-soak-testing-and-workload-modeling.md`, `26-finops-and-cloud-cost.md` |
|
|
28
|
+
| High-level architecture, boundaries, synchronous flows or service decomposition | `02-architecture-and-networking.md` | `13-architecture-intelligence.md`, `25-platform-engineering-and-developer-experience.md` |
|
|
29
|
+
| Repository architecture review | `13-architecture-intelligence.md` | `15-failure-analysis.md`, then domain-specific references identified from inspected code |
|
|
30
|
+
| Relational schema, query plans, indexes or engine behavior | `03-data-and-storage.md` | PostgreSQL: `34-postgresql-internals-and-performance.md`; MySQL: `35-mysql-innodb-and-performance.md`; partitioning: `36-sharding-and-distributed-sql.md`; storage media: `67-storage-infrastructure-and-storage-systems.md` |
|
|
31
|
+
| Consistency, consensus, leader election, fencing or distributed locks | `06-distributed-systems.md` | `37-consensus-and-distributed-transactions.md`, `03-data-and-storage.md` for database isolation/replication, `24-multi-region-and-disaster-recovery.md` for regional ownership |
|
|
32
|
+
| Cross-service business transaction, saga, 2PC or reconciliation | `37-consensus-and-distributed-transactions.md` | `05-async-and-messaging.md`, `29-event-sourcing-and-cqrs.md`, `06-distributed-systems.md` |
|
|
33
|
+
| Database schema migration or zero-downtime rollout | `40-zero-downtime-database-migrations.md` | Engine-specific guidance in `34-postgresql-internals-and-performance.md` or `35-mysql-innodb-and-performance.md`; CDC: `30-change-data-capture-and-debezium.md`; event compatibility: `33-schema-registry-and-event-governance.md` |
|
|
34
|
+
| Cache behavior, invalidation, stampede or stale content | `04-caching.md` | `03-data-and-storage.md` for source-of-truth/replica semantics; `07-reliability-and-resilience.md`, `64-cdn-and-edge-caching.md` |
|
|
35
|
+
| Queueing, delivery guarantees, backpressure or consumer failure | `05-async-and-messaging.md` | Kafka/Pulsar: `28-event-streaming-and-kafka-pulsar.md`; workflow: `32-workflow-orchestration-and-temporal.md`; schema: `33-schema-registry-and-event-governance.md` |
|
|
36
|
+
| Kafka/Pulsar partitioning, replication, offsets or broker operations | `28-event-streaming-and-kafka-pulsar.md` | Generic delivery semantics: `05-async-and-messaging.md`; CDC: `30-change-data-capture-and-debezium.md`; event schemas: `33-schema-registry-and-event-governance.md` |
|
|
37
|
+
| Durable workflow state, activity replay, timers or cancellation | `32-workflow-orchestration-and-temporal.md` | Queue delivery: `05-async-and-messaging.md`; cross-resource guarantees: `37-consensus-and-distributed-transactions.md` |
|
|
38
|
+
| Event sourcing, CQRS, projections or replay | `29-event-sourcing-and-cqrs.md` | `05-async-and-messaging.md`, `28-event-streaming-and-kafka-pulsar.md`, `33-schema-registry-and-event-governance.md` |
|
|
39
|
+
| CDC, database log capture or stream/batch data platform | `30-change-data-capture-and-debezium.md` | `31-data-lakehouse-and-stream-batch.md`, `33-schema-registry-and-event-governance.md` |
|
|
40
|
+
| Reliability, incident, recovery or failover | `07-reliability-and-resilience.md` | `15-failure-analysis.md`, `24-multi-region-and-disaster-recovery.md`, `60-chaos-engineering-and-resilience-experiments.md`, `61-incident-management-and-postmortems.md` |
|
|
41
|
+
| Planned fault injection or resilience experiment design | `60-chaos-engineering-and-resilience-experiments.md` | Underlying failure modes: `15-failure-analysis.md`; service controls: `07-reliability-and-resilience.md` |
|
|
42
|
+
| Live incident command, communications or postmortem | `61-incident-management-and-postmortems.md` | Technical failure analysis: `15-failure-analysis.md`; reliability controls: `07-reliability-and-resilience.md` |
|
|
43
|
+
| SLO, telemetry, tracing or production diagnostics | `27-production-observability-and-slo.md` | `55-opentelemetry-and-instrumentation.md`, `56-distributed-tracing-and-context-propagation.md`, `57-continuous-profiling-and-performance-diagnostics.md` |
|
|
44
|
+
| Trace context, baggage, sampling or cross-service latency interpretation | `56-distributed-tracing-and-context-propagation.md` | Instrumentation pipeline: `55-opentelemetry-and-instrumentation.md`; SLO/alerting: `27-production-observability-and-slo.md`; privacy policy: `54-privacy-engineering-and-data-governance.md` |
|
|
45
|
+
| API design, schema or data contract | `14-api-and-data-contracts.md` | REST: `42-rest-api-design-and-lifecycle.md`; GraphQL: `43-graphql-architecture-and-governance.md`; gRPC: `44-grpc-and-service-contracts.md`; gateway: `41-api-gateway-and-edge-architecture.md` |
|
|
46
|
+
| API authentication, authorization, abuse or rate limiting | `47-api-security-rate-limiting-and-abuse.md` | OAuth/OIDC: `48-oauth-oidc-and-passkeys.md`; IAM: `49-iam-rbac-abac-rebac.md`; threat modeling: `53-threat-modeling-stride-and-attack-trees.md` |
|
|
47
|
+
| Identity, workload trust, secrets or key management | `08-api-security-observability.md` | `48-oauth-oidc-and-passkeys.md`, `49-iam-rbac-abac-rebac.md`, `50-zero-trust-and-workload-identity.md`, `51-secrets-kms-pki-and-key-management.md` |
|
|
48
|
+
| Supply-chain security, privacy or threat modeling | `53-threat-modeling-stride-and-attack-trees.md` | Supply chain: `52-supply-chain-security-sbom-slsa.md`; privacy: `54-privacy-engineering-and-data-governance.md`; API abuse: `47-api-security-rate-limiting-and-abuse.md` |
|
|
49
|
+
| WebSocket, SSE or bidirectional real-time delivery | `45-websocket-sse-and-realtime-systems.md` | Collaboration state: `46-realtime-collaboration-ot-crdt.md`; API contracts: `14-api-and-data-contracts.md` |
|
|
50
|
+
| Search, indexing or retrieval | `38-search-architecture-elasticsearch-opensearch.md` | Vector/hybrid retrieval: `39-vector-databases-and-hybrid-retrieval.md` |
|
|
51
|
+
| AI application, agent or model-serving architecture | `12-ai-systems-architecture.md` | Multi-agent: `16-multi-agent-architecture.md`; harness/runtime: `17-agent-harness-and-runtime.md`; MCP: `18-mcp-architecture.md`; agent state: `20-agent-state-and-memory.md` |
|
|
52
|
+
| Agent authorization, tool permissions or prompt/tool boundary | `19-agent-security-and-permissions.md` | `17-agent-harness-and-runtime.md`, `18-mcp-architecture.md`, `50-zero-trust-and-workload-identity.md` |
|
|
53
|
+
| Agent outbound disclosure, tool-result injection, or approval-scope drift | `19-agent-security-and-permissions.md` | `17-agent-harness-and-runtime.md`, `18-mcp-architecture.md`, `54-privacy-engineering-and-data-governance.md`, `50-zero-trust-and-workload-identity.md` |
|
|
54
|
+
| Indirect prompt injection or untrusted retrieved/tool content | `19-agent-security-and-permissions.md` | `18-mcp-architecture.md`, `17-agent-harness-and-runtime.md`, `50-zero-trust-and-workload-identity.md` |
|
|
55
|
+
| Agent checkpoint, resume, or ambiguous tool side effect | `20-agent-state-and-memory.md` | `17-agent-harness-and-runtime.md`, `37-consensus-and-distributed-transactions.md`, `15-failure-analysis.md` |
|
|
56
|
+
| Incident recovery, queue backlog, or safe replay | `61-incident-management-and-postmortems.md` | `60-chaos-engineering-and-resilience-experiments.md`, `05-async-and-messaging.md`, `37-consensus-and-distributed-transactions.md` |
|
|
57
|
+
| Agent evaluation, traces or quality regression | `21-agent-evaluation-and-observability.md` | `12-ai-systems-architecture.md`, `27-production-observability-and-slo.md` |
|
|
58
|
+
| Cloud platform, deployment engineering or broad infrastructure practices | `09-modern-engineering.md` | Select the relevant Kubernetes, platform engineering, FinOps, networking, or observability reference based on the concrete constraint |
|
|
59
|
+
| Kubernetes, containers or service mesh | `22-kubernetes-and-container-platforms.md` | Service mesh: `23-service-mesh-and-traffic-management.md`; platform: `25-platform-engineering-and-developer-experience.md` |
|
|
60
|
+
| DNS, load balancing, global routing or cloud connectivity | `62-dns-and-service-discovery.md` | Load balancing: `63-load-balancing-and-global-traffic-management.md`; cloud networking: `65-cloud-networking-and-connectivity.md`; TCP/QUIC: `68-tcp-quic-and-network-performance.md` |
|
|
61
|
+
| CDN, edge delivery or edge compute | `64-cdn-and-edge-caching.md` | Edge compute: `66-edge-computing-architecture.md`; global routing: `63-load-balancing-and-global-traffic-management.md` |
|
|
62
|
+
| Performance optimization or capacity bottleneck | `57-continuous-profiling-and-performance-diagnostics.md` | Load testing: `58-load-stress-soak-testing-and-workload-modeling.md`; capacity model: `59-capacity-planning-and-performance-modeling.md`; database-specific references when the bottleneck is storage |
|
|
63
|
+
| Architecture choice or ADR | `13-architecture-intelligence.md` | Relevant domain reference plus `templates/architecture-decision-matrix-template.md` |
|
|
64
|
+
| Interview practice, design workflow or review checklist | `10-workflows-and-checklists.md` | `01-requirements-and-capacity.md`, then the domain-specific reference for the prompt |
|
|
65
|
+
| Worked architecture examples or system case studies | `11-case-studies.md` | `01-requirements-and-capacity.md` plus the relevant storage, messaging, reliability, or API references |
|
|
66
|
+
| Primer concept lookup or attribution | `00-primer-map-and-attribution.md` | Follow the mapped modern reference for production-specific details |
|
|
67
|
+
|
|
68
|
+
## Cross-domain selection rules
|
|
69
|
+
|
|
70
|
+
- **Respect specialist ownership boundaries.** Use `05-async-and-messaging.md` for broker-agnostic delivery fundamentals and `28-event-streaming-and-kafka-pulsar.md` for Kafka/Pulsar-specific behavior; use `32-workflow-orchestration-and-temporal.md` for durable process execution; use `37-consensus-and-distributed-transactions.md` for transaction guarantees. For resilience, distinguish service design (`07-reliability-and-resilience.md`), failure-analysis method (`15-failure-analysis.md`), planned experiments (`60-chaos-engineering-and-resilience-experiments.md`), and live incident operations (`61-incident-management-and-postmortems.md`). These documents intentionally cross-link; do not treat repeated terms as interchangeable ownership.
|
|
71
|
+
|
|
72
|
+
- **Transactions are not just a database topic.** For a business invariant crossing services or resources, combine the transaction/consensus reference with the actual delivery mechanism (outbox, queue, workflow, or CDC) and the domain's reconciliation requirements.
|
|
73
|
+
- **A migration is not just DDL.** Include application-version compatibility, backfill behavior, replication/CDC impact, validation, cutover, and rollback. Load engine-specific references when lock or online-DDL behavior matters.
|
|
74
|
+
- **Security is not one checklist.** Select the identity, API-abuse, workload-trust, supply-chain, or privacy reference matching the trust boundary under review.
|
|
75
|
+
- **Consistency terms must be scoped.** Distinguish linearizability, serializability, session guarantees, eventual convergence, and replica freshness; state the data boundary and failure assumptions rather than using “strong” or “consistent” without a contract.
|
|
76
|
+
- **Version-sensitive claims need a version and source.** For database isolation/locking, Kafka producer and transaction guarantees, OIDC token validation, Kubernetes component skew, and transport protocol behavior, state the tested product/version and link the relevant official specification or vendor documentation. Treat defaults as versioned configuration, not timeless facts; re-check them before deployment or upgrade.
|
|
77
|
+
- **Cache correctness crosses layers.** Identify the authoritative store, every cache/replica layer, invalidation path, stale-read tolerance, and recovery behavior; TTL alone is not a consistency protocol.
|
|
78
|
+
- **Capacity is end-to-end.** Follow the constrained dependency and model queue stability, failure headroom, workload mix, and cost; do not size only the stateless application tier.
|
|
79
|
+
- **Reliability requires recovery evidence.** Pair design guidance with failure analysis, restore/failover exercises, and incident learning when the task concerns production readiness.
|
|
80
|
+
- Do not load every file listed in a row automatically. Add a reference only when its topic changes the decision or validates a material risk.
|
|
81
|
+
|
|
82
|
+
## Evidence protocol for repository reviews
|
|
83
|
+
|
|
84
|
+
1. **Declare scope.** List inspected directories, entry points, configuration, tests, deployment files, and known exclusions. State whether the review is static-only.
|
|
85
|
+
2. **Trace critical paths.** Follow at least one important request or event from entry point through authorization, domain logic, persistence/external calls, and response/event publication.
|
|
86
|
+
3. **Map dependencies.** Record caller/callee, data ownership, sync/async boundaries, and trust boundaries. Draw only edges supported by inspected code or configuration.
|
|
87
|
+
4. **Classify evidence.** Use Confirmed for directly observed code/configuration, Likely for a reasoned inference, and Unknown when runtime or repository visibility is insufficient.
|
|
88
|
+
5. **Prioritize by impact.** Rank data loss/corruption, security exposure, unavailability, and uncontrolled cost above style or speculative scale concerns.
|
|
89
|
+
6. **Make findings reproducible.** Include file path and symbol/config key, trigger conditions, impact, recommended change, and a validation method.
|
|
90
|
+
7. **State limitations.** Static inspection does not establish production traffic, deployed topology, live configuration, incident history, or actual performance. Request runtime metrics or traces when those facts affect the conclusion.
|
|
91
|
+
|
|
92
|
+
## Reference-loading rules
|
|
93
|
+
|
|
94
|
+
- Read this routing index first, then the smallest relevant reference set.
|
|
95
|
+
- Prefer the exact technology or failure-domain reference over loading broad adjacent topics.
|
|
96
|
+
- Load templates only when producing that deliverable; do not force a template into a short answer.
|
|
97
|
+
- If a task spans domains, state why each additional reference is needed.
|
|
98
|
+
- Verify fast-changing vendor, product, and protocol details against current official documentation; these references are engineering guidance, not live product specifications.
|
|
99
|
+
- Check neighboring references for conflicting terminology or duplicated advice when a decision crosses domains. Resolve conflicts by stating assumptions and the specific system/version behavior that controls.
|
|
100
|
+
- Do not treat a reference's presence as evidence that a repository implements the described mechanism.
|
|
101
|
+
|
|
102
|
+
## Confidence and estimate labels
|
|
103
|
+
|
|
104
|
+
Use these labels consistently:
|
|
105
|
+
|
|
106
|
+
- **Observed:** directly visible in source, configuration, tests, or supplied measurements.
|
|
107
|
+
- **Inferred:** plausible conclusion from observed evidence; state the reasoning and alternatives.
|
|
108
|
+
- **Assumed:** temporary input used to make progress; identify the owner or question that can confirm it.
|
|
109
|
+
- **Unknown:** unavailable from current inspection; name the evidence needed to resolve it.
|
|
110
|
+
|
|
111
|
+
For capacity and cost estimates, show the formula, units, input provenance, and sensitivity range. Never convert an assumed peak factor or average record size into a measured fact.
|
|
112
|
+
|
|
113
|
+
## Regression cases
|
|
114
|
+
|
|
115
|
+
The machine-readable cases in `evals/system-design-cases.json` cover representative routing and output expectations. They are prompt fixtures for human or model-assisted evaluation, not deterministic proof of answer quality. Evaluate both the selected mode/references and whether the response respects evidence, safety, and scope guardrails.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Seal or verify versioned SHA-256 hash chains for evaluation JSONL records."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
try:
|
|
9
|
+
from .evaluate import seal_audit_chain, verify_audit_chain
|
|
10
|
+
except ImportError:
|
|
11
|
+
from evaluate import seal_audit_chain, verify_audit_chain
|
|
12
|
+
|
|
13
|
+
def main() -> int:
|
|
14
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
15
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
16
|
+
seal = sub.add_parser("seal", help="Create a new hash-chained JSONL snapshot")
|
|
17
|
+
seal.add_argument("--input", type=Path, required=True)
|
|
18
|
+
seal.add_argument("--output", type=Path, required=True)
|
|
19
|
+
verify = sub.add_parser("verify", help="Verify an existing hash-chained JSONL file")
|
|
20
|
+
verify.add_argument("--input", type=Path, required=True)
|
|
21
|
+
verify.add_argument("--expected-head-hash", help="Optional independently retained head hash")
|
|
22
|
+
args = parser.parse_args()
|
|
23
|
+
try:
|
|
24
|
+
if args.command == "seal":
|
|
25
|
+
count = seal_audit_chain(args.input, args.output)
|
|
26
|
+
result = {"sealed": True, "records": count, "output": str(args.output)}
|
|
27
|
+
else:
|
|
28
|
+
result = verify_audit_chain(args.input, args.expected_head_hash)
|
|
29
|
+
except (OSError, ValueError) as exc:
|
|
30
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
31
|
+
return 1
|
|
32
|
+
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
33
|
+
return 0
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fail-closed comparability check for two evaluation reports."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import argparse, json, sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
FIELDS=("system_prompt_sha256","tool_permission_profile","run_config_id")
|
|
9
|
+
def load(path: Path) -> dict[str,Any]:
|
|
10
|
+
value=json.loads(path.read_text(encoding="utf-8"))
|
|
11
|
+
if not isinstance(value,dict): raise ValueError(f"{path}: expected JSON object")
|
|
12
|
+
return value
|
|
13
|
+
|
|
14
|
+
def compare_reports(left: dict[str,Any], right: dict[str,Any]) -> dict[str,Any]:
|
|
15
|
+
a,b=left.get("evaluation_catalog"),right.get("evaluation_catalog")
|
|
16
|
+
if not isinstance(a,dict) or not isinstance(b,dict):
|
|
17
|
+
return {"status":"unknown","comparable":False,"reason":"catalog metadata missing (legacy report)"}
|
|
18
|
+
required=("dataset_canonical_sha256","rubric_sha256","case_ids")
|
|
19
|
+
missing={"left":[k for k in required if a.get(k) is None],"right":[k for k in required if b.get(k) is None]}
|
|
20
|
+
if any(missing.values()):
|
|
21
|
+
return {"status":"unknown","comparable":False,"reason":"dataset/rubric fingerprints incomplete","missing_fields":missing}
|
|
22
|
+
diffs=[]
|
|
23
|
+
for k in required:
|
|
24
|
+
if a[k]!=b[k]: diffs.append({"field":k,"left":a[k],"right":b[k]})
|
|
25
|
+
pa,pb=left.get("evaluation_provenance",{}),right.get("evaluation_provenance",{})
|
|
26
|
+
unknown=[]
|
|
27
|
+
for k in FIELDS:
|
|
28
|
+
x,y=pa.get(k),pb.get(k)
|
|
29
|
+
if x is None or y is None: unknown.append(k)
|
|
30
|
+
elif x!=y: diffs.append({"field":"evaluation_provenance."+k,"left":x,"right":y})
|
|
31
|
+
if diffs: return {"status":"incompatible","comparable":False,"reason":"dataset, rubric, or evaluation context differs","differences":diffs,"unknown_fields":unknown}
|
|
32
|
+
if unknown: return {"status":"unknown","comparable":False,"reason":"required evaluation context missing","unknown_fields":unknown}
|
|
33
|
+
return {"status":"comparable","comparable":True,"reason":"dataset, rubric, prompt, tool permissions, and run configuration match",
|
|
34
|
+
"dataset_canonical_sha256":a["dataset_canonical_sha256"],"rubric_sha256":a["rubric_sha256"],
|
|
35
|
+
"case_count":len(a["case_ids"]),"model_labels":{"left":pa.get("model_labels",[]),"right":pb.get("model_labels",[])},
|
|
36
|
+
"note":"Design comparability only; this does not establish statistical significance or model quality."}
|
|
37
|
+
|
|
38
|
+
def main()->int:
|
|
39
|
+
p=argparse.ArgumentParser(description=__doc__); p.add_argument("--left-report",type=Path,required=True); p.add_argument("--right-report",type=Path,required=True); a=p.parse_args()
|
|
40
|
+
try: result=compare_reports(load(a.left_report),load(a.right_report))
|
|
41
|
+
except (OSError,json.JSONDecodeError,ValueError) as e: print(f"ERROR: {e}",file=sys.stderr); return 1
|
|
42
|
+
print(json.dumps(result,ensure_ascii=False,sort_keys=True,indent=2)); return 0 if result["comparable"] else 2
|
|
43
|
+
if __name__=="__main__": raise SystemExit(main())
|