replay-ci 1.0.0

package/README.md

@@ -0,0 +1,5 @@
+# ReplayCI
+
+Milestone work lives in:
+- packs/slack-v0.1 (contracts + golden inputs)
+- evidence/Q1-M1 (proof artifacts)

package/artifacts/schema/run.schema.json

@@ -0,0 +1,445 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "PacketDesk Run Artifact v0.2",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "traceability",
+    "run_policy",
+    "state",
+    "steps"
+  ],
+  "properties": {
+    "schema_version": {
+      "type": "string",
+      "enum": [
+        "0.2"
+      ]
+    },
+    "traceability": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "contract_hash",
+        "policy_hash",
+        "pack_version",
+        "runner_version",
+        "env_hash",
+        "run_mode",
+        "schema_version",
+        "tenant_id"
+      ],
+      "properties": {
+        "contract_hash": {
+          "type": "string"
+        },
+        "policy_hash": {
+          "type": "string"
+        },
+        "pack_version": {
+          "type": "string"
+        },
+        "runner_version": {
+          "type": "string"
+        },
+        "env_hash": {
+          "type": "string"
+        },
+        "run_mode": {
+          "type": "string",
+          "enum": [
+            "manual",
+            "scheduled",
+            "ci"
+          ]
+        },
+        "schema_version": {
+          "type": "string",
+          "enum": [
+            "0.2"
+          ]
+        },
+        "tenant_id": {
+          "type": "string"
+        },
+        "provider": {
+          "type": "string",
+          "description": "Provider used for this run (openai|anthropic|recorded). Optional at envelope level; set per-step by adapters."
+        },
+        "model_id": {
+          "type": "string",
+          "description": "Model identifier used for this run. Optional at envelope level; set per-step by adapters."
+        },
+        "model_version": {
+          "type": "string",
+          "description": "Best-effort model version string returned by provider. Optional."
+        },
+        "normalization_profile": {
+          "type": "string",
+          "enum": ["strict_hash", "evidence_diff"],
+          "description": "Default normalization profile for this run."
+        },
+        "corpus_manifest_hash": {
+          "type": "string",
+          "pattern": "^[a-f0-9]{64}$",
+          "description": "Phase 2 (GR-05): SHA-256 hash of the deterministic corpus. Merge gate runs against pinned manifest only."
+        }
+      }
+    },
+    "run_policy": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "run_mode",
+        "timeout_ms",
+        "retry_cap",
+        "max_contracts"
+      ],
+      "properties": {
+        "run_mode": {
+          "type": "string",
+          "enum": [
+            "manual",
+            "scheduled",
+            "ci"
+          ]
+        },
+        "timeout_ms": {
+          "type": "integer",
+          "minimum": 1000,
+          "maximum": 120000
+        },
+        "retry_cap": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 5
+        },
+        "max_contracts": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 50
+        },
+        "only_contracts": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "side_effect_mode": {
+          "type": "string",
+          "enum": [
+            "read_only",
+            "allow_all"
+          ]
+        }
+      }
+    },
+    "state": {
+      "type": "string",
+      "enum": [
+        "Pass",
+        "Fail",
+        "NonReproducible",
+        "RunAborted"
+      ]
+    },
+    "abort_reason": {
+      "oneOf": [
+        {
+          "type": "null"
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "code"
+          ],
+          "properties": {
+            "code": {
+              "type": "string",
+              "enum": [
+                "timeout",
+                "budget_exceeded",
+                "manual_cancel"
+              ]
+            },
+            "detail": {
+              "type": "string"
+            }
+          }
+        }
+      ]
+    },
+    "steps": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "$comment": "normalization_profile strict_hash evidence_diff raw_redacted normalized derived_metrics",
+        "required": [
+          "index",
+          "contract_path",
+          "status",
+          "nonrepro_reason",
+          "raw_redacted",
+          "normalized",
+          "derived_metrics",
+          "fingerprint"
+        ],
+        "properties": {
+          "index": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "contract_path": {
+            "type": "string"
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "Pass",
+              "Fail",
+              "NonReproducible",
+              "RunAborted"
+            ]
+          },
+          "raw_redacted": {},
+          "normalized": {},
+          "derived_metrics": {
+            "type": "object",
+            "additionalProperties": true,
+            "properties": {
+              "retry": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "retry_amplification",
+                  "attempts",
+                  "stop_reason"
+                ],
+                "properties": {
+                  "retry_amplification": {
+                    "type": "integer",
+                    "minimum": 0
+                  },
+                  "stop_reason": {
+                    "type": "string"
+                  },
+                  "retry_after_header": {
+                    "type": [
+                      "string",
+                      "null"
+                    ]
+                  },
+                  "retry_after_seconds": {
+                    "type": [
+                      "integer",
+                      "null"
+                    ],
+                    "minimum": 0
+                  },
+                  "attempts": {
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "additionalProperties": false,
+                      "required": [
+                        "attempt",
+                        "decision",
+                        "reason"
+                      ],
+                      "properties": {
+                        "attempt": {
+                          "type": "integer",
+                          "minimum": 1
+                        },
+                        "http_status": {
+                          "type": [
+                            "integer",
+                            "null"
+                          ]
+                        },
+                        "decision": {
+                          "type": "string",
+                          "enum": [
+                            "retry",
+                            "wait",
+                            "stop"
+                          ]
+                        },
+                        "reason": {
+                          "type": "string"
+                        },
+                        "retry_after_header": {
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        },
+                        "retry_after_seconds": {
+                          "type": [
+                            "integer",
+                            "null"
+                          ],
+                          "minimum": 0
+                        },
+                        "wait_ms": {
+                          "type": [
+                            "integer",
+                            "null"
+                          ],
+                          "minimum": 0
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "normalization_profile": {
+            "type": "string",
+            "enum": [
+              "strict_hash",
+              "evidence_diff"
+            ],
+            "description": "normalization_profile selects the normalization semantics profile: strict_hash or evidence_diff."
+          },
+          "strict_hash": {
+            "type": "string",
+            "pattern": "^[A-Fa-f0-9]{64}$",
+            "description": "strict_hash is sha256(canonical_json(normalized)); legacy fingerprint remains supported as an alias/legacy field."
+          },
+          "fingerprint": {
+            "type": "string",
+            "pattern": "^[a-f0-9]{64}$"
+          },
+          "evidence_diff": {
+            "description": "Redacted evidence payload for human-readable drift analysis. Derived from raw_redacted."
+          },
+          "baseline_key": {
+            "type": "string",
+            "pattern": "^[a-f0-9]{64}$",
+            "description": "Phase 2 (GR-20): SHA-256 of (contract_path:normalization_profile). Incompatible baselines cannot mix."
+          },
+          "nr_allowlisted": {
+            "type": "boolean",
+            "description": "GR-07: Whether this NR step is explicitly allowlisted. Only set when status is NonReproducible."
+          },
+          "nonrepro_reason": {
+            "oneOf": [
+              {
+                "type": "null"
+              },
+              {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "code"
+                ],
+                "properties": {
+                  "code": {
+                    "type": "string",
+                    "enum": [
+                      "model_nondeterminism",
+                      "context_variance",
+                      "transient_error",
+                      "MISSING_PREREQUISITES",
+                      "NON_DETERMINISTIC_INPUT",
+                      "SCHEMA_DRIFT",
+                      "UNKNOWN",
+                      "UPSTREAM_RATE_LIMIT",
+                      "UPSTREAM_TIMEOUT",
+                      "UPSTREAM_TRANSIENT_ERROR"
+                    ]
+                  },
+                  "detail": {
+                    "type": "string"
+                  }
+                }
+              }
+            ]
+          },
+          "failure_classifier": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "category",
+              "stable_signature",
+              "confidence",
+              "is_non_reproducible",
+              "fingerprint",
+              "schema_version"
+            ],
+            "properties": {
+              "category": {
+                "type": "string",
+                "enum": [
+                  "auth",
+                  "rate_limit",
+                  "upstream",
+                  "schema_payload",
+                  "deprecation",
+                  "unknown",
+                  "non_reproducible"
+                ]
+              },
+              "stable_signature": {
+                "type": "string"
+              },
+              "confidence": {
+                "type": "number"
+              },
+              "is_non_reproducible": {
+                "type": "boolean"
+              },
+              "non_reproducible_reason": {
+                "type": "string",
+                "enum": [
+                  "upstream_jitter",
+                  "environment_mismatch",
+                  "missing_prerequisites",
+                  "timeout"
+                ]
+              },
+              "fingerprint": {
+                "type": "string"
+              },
+              "schema_version": {
+                "type": "string"
+              },
+              "taxonomy_version": {
+                "type": "string"
+              },
+              "global_key": {
+                "type": "string"
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "state": {
+            "const": "RunAborted"
+          }
+        }
+      },
+      "then": {
+        "required": [
+          "abort_reason"
+        ],
+        "properties": {
+          "abort_reason": {
+            "type": "object"
+          }
+        }
+      }
+    }
+  ]
+}

package/bin/replayci.mjs

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+// Thin shim: registers tsx for TypeScript support, then runs the CLI.
+// Enables `npx replayci` without `npm run replayci --`.
+import { register } from "node:module";
+import { pathToFileURL } from "node:url";
+
+register("tsx/esm", pathToFileURL("./"));
+await import("../src/cli/main.ts");

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "replay-ci",
+  "version": "1.0.0",
+  "description": "LLM function calling reliability platform. Validate, monitor, and gate tool-call behavior.",
+  "main": "index.js",
+  "bin": {
+    "replayci": "./bin/replayci.mjs"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "packs/starter/",
+    "tsconfig.json",
+    "tsconfig.container.json",
+    "artifacts/schema/"
+  ],
+  "directories": {
+    "doc": "docs"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "lint": "tsc -p tsconfig.lint.json --noEmit",
+    "replayci": "tsx src/cli/main.ts",
+    "dashboard:dev": "next dev -p 3001 --webpack",
+    "dashboard:build": "next build --webpack",
+    "dashboard:start": "next start -p 3001"
+  },
+  "keywords": [
+    "llm",
+    "testing",
+    "tool-calling",
+    "function-calling",
+    "openai",
+    "anthropic",
+    "ci",
+    "contracts"
+  ],
+  "author": "ReplayCI",
+  "license": "ISC",
+  "type": "module",
+  "homepage": "https://replayci.com",
+  "devDependencies": {
+    "@types/node": "^25.0.3",
+    "@types/nodemailer": "^7.0.11",
+    "@types/pg": "^8.16.0",
+    "@types/pino": "^7.0.4",
+    "@types/react": "^19.2.7",
+    "@types/react-dom": "^19.2.3",
+    "pino-pretty": "^13.1.3",
+    "playwright": "^1.57.0",
+    "prisma": "^7.2.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
+  },
+  "dependencies": {
+    "@prisma/adapter-pg": "^7.2.0",
+    "@prisma/client": "^7.2.0",
+    "@tailwindcss/postcss": "^4.2.1",
+    "ajv": "^8.17.1",
+    "next": "^16.1.1",
+    "nodemailer": "^8.0.1",
+    "pg": "^8.16.3",
+    "pino": "^10.3.1",
+    "react": "^19.2.3",
+    "react-dom": "^19.2.3",
+    "tailwindcss": "^4.2.1",
+    "tsx": "^4.21.0",
+    "yaml": "^2.8.2"
+  }
+}

package/packs/starter/contracts/tool_call.yaml

@@ -0,0 +1,23 @@
+# Does my AI call the right tool?
+# This contract validates that the model invokes the expected tool
+# with valid arguments when given a tool-calling prompt.
+
+tool: "get_weather"
+
+assertions:
+  output_invariants:
+    - path: "$.tool_calls[0].name"
+      equals: "get_weather"
+    - path: "$.tool_calls[0].arguments"
+      exists: true
+
+golden_cases:
+  - id: "tool_call_success"
+    input_ref: "tool_call.success.json"
+    expect_ok: true
+
+  - id: "tool_not_invoked"
+    input_ref: "negative/tool_call.not_invoked.json"
+    expect_ok: false
+    expected_error: "tool_not_invoked"
+    provider_modes: ["recorded"]

package/packs/starter/golden/negative/tool_call.not_invoked.json

@@ -0,0 +1,41 @@
+{
+  "boundary": {
+    "model": "gpt-4o-mini",
+    "tool_schema_hash": "starter_tool_neg001",
+    "message_hash": "starter_msg_neg001",
+    "system_hash": "starter_sys_neg001"
+  },
+  "request": {
+    "model": "gpt-4o-mini",
+    "messages": [
+      {
+        "role": "user",
+        "content": "What is the weather in Tokyo?"
+      }
+    ],
+    "tools": [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "description": "Get the current weather for a location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string"
+              }
+            },
+            "required": ["location"]
+          }
+        }
+      }
+    ]
+  },
+  "response": {
+    "success": false,
+    "tool_calls": [],
+    "content": "I don't have access to real-time weather data.",
+    "error_code": "tool_not_invoked"
+  }
+}

package/packs/starter/golden/tool_call.success.json

@@ -0,0 +1,54 @@
+{
+  "boundary": {
+    "model": "gpt-4o-mini",
+    "tool_schema_hash": "starter_tool_001",
+    "message_hash": "starter_msg_001",
+    "system_hash": "starter_sys_001"
+  },
+  "request": {
+    "model": "gpt-4o-mini",
+    "messages": [
+      {
+        "role": "system",
+        "content": "You are a helpful assistant with access to tools."
+      },
+      {
+        "role": "user",
+        "content": "What is the weather in San Francisco?"
+      }
+    ],
+    "tools": [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "description": "Get the current weather for a location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string",
+                "description": "City and state, e.g. San Francisco, CA"
+              }
+            },
+            "required": ["location"]
+          }
+        }
+      }
+    ]
+  },
+  "response": {
+    "success": true,
+    "tool_calls": [
+      {
+        "id": "call_starter_001",
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "arguments": "{\"location\": \"San Francisco, CA\"}"
+        }
+      }
+    ],
+    "content": null
+  }
+}

package/packs/starter/nr-allowlist.json

@@ -0,0 +1,5 @@
+{
+  "description": "Starter pack NR allowlist. Empty — no exceptions needed for starter contracts.",
+  "version": "1.0",
+  "entries": []
+}

package/packs/starter/pack.yaml

@@ -0,0 +1,19 @@
+# ReplayCI Starter Pack
+# Minimal contract pack for getting started with tool-call validation.
+# Customize contracts and golden fixtures for your own tools.
+
+pack_id: "starter"
+name: "starter-tool-calling"
+version: "0.1.0"
+schema_version: "v0.1"
+
+provider: "openai"
+default_model: "gpt-4o-mini"
+
+paths:
+  contracts_dir: "packs/starter/contracts"
+  golden_dir: "packs/starter/golden"
+  negative_golden_dir: "packs/starter/golden/negative"
+
+contracts:
+  - tool_call.yaml

package/src/alerting/DEPRECATED.md

@@ -0,0 +1,7 @@
+# DEPRECATED
+
+This directory contains the legacy Slack alerting system from ReplayCI's pre-pivot plan (MCP/Slack reliability tool). It is no longer part of the active product.
+
+**Do not use or extend this code.** It remains in the repository for reference only.
+
+The current product is an LLM Function Calling Reliability Platform. See `CLAUDE.md` for the current architecture.