securenow 8.21.0 → 8.22.0

package/SKILL-CLI.md

@@ -398,6 +398,7 @@ securenow alerts rules change <id> "raise the threshold to 20"   # rewrite a rul
 securenow alerts rules runs <id> [--since 1h] # what each run saw and why it did or didn't notify (+ next scheduled run)
 securenow alerts rules run <id>               # run a periodic rule now and print the run (--no-wait to just queue it)
 securenow alerts rules why <id> --ip <ip> [--since 1h]   # why the rule did or didn't alert on one address: verdict + findings
+securenow alerts rules quality <id>           # was the rule right to fire? 14-day precision, alerts labeled wrong/unclear and why, what detection learning changed
 securenow alerts rules dry-run-query <id> --sql @candidate.sql --app <key> --wait   # candidate test against a specific host rule
 securenow alerts rules tune-query <id> --sql @candidate.sql --reason "Preserve exploit detector, remove benign broad match" --apply-globally --yes
 securenow alerts rules exclusions <id> list              # embedded rule exclusions
@@ -487,10 +488,14 @@ securenow protect enable --app <key>      # turn it on (monitor mode)
 securenow protect block --app <key>       # a person turns blocking on; refused unless the app sees real client IPs
 securenow protect monitor --app <key>     # back to alerts only
 securenow protect disable --app <key>     # off; stays off until someone turns it on
+securenow protect learn --app <key>       # detection learning now (it also runs hourly): label alerts, measure precision, learn safe exclusions
+securenow protect status --rules          # adds each rule's 14-day precision (right / right + wrong)
 ```
 
 Only a person signed in (`securenow admin login`) can turn blocking on; API keys, MCP connectors and the agent get a 403. Blocking is refused when more than 20% of the app's traffic reaches it through a proxy the SDK doesn't trust: add the proxy to `config.networking.trustedProxies` (IPs, IPv4 ranges or `"cloudflare"`). Tune a noisy starter rule like any rule (exclusions, `alerts rules update`); pack updates never overwrite a rule a person edited.
 
+**Detection learning** keeps the starter rules precise per app. Every alert gets a label: was the rule right to fire? A blocked address counts as right, a person's false-positive call as wrong, and an LLM judge (accounts with AI access) labels the rest from the requests and the address's last day of traffic; a failed one-off probe (`/.env` answered 404) is closed as right but not worth action. Precision = right / (right + wrong) over 14 days, target 95%. A rule below it with 3+ wrong alerts gets one exclusion for the legitimate traffic, applied only when it matches the wrong alerts and none of the right ones, can't be pinned to an IP, and still lets every reference attack through even when injected into the excused request; otherwise the rule is flagged for a person. A block-ready rule under 95% (10+ labeled alerts) keeps the app from being ready to block. MCP: `securenow_alert_rule_quality`, `securenow_protection_learn`.
+
 ### Blocklist — Block Malicious IPs
 
 ```bash

package/cli/protect.js

@@ -31,13 +31,27 @@ function modeLabel(app) {
   return ui.c.yellow(app.state === 'unset' ? 'NOT YET ON' : 'OFF');
 }
 
+// Precision: of the alerts labeled right or wrong (detection learning), the share where the rule was right.
+const PRECISION_TARGET = 0.95;
+
+function precisionCell(quality) {
+  if (!quality || quality.precision == null) return ui.c.dim('-');
+  const decided = quality.decided ?? (quality.correct || 0) + (quality.wrong || 0);
+  const text = `${(quality.precision * 100).toFixed(quality.precision === 1 ? 0 : 1)}% (${quality.correct}/${decided})`;
+  return quality.precision >= PRECISION_TARGET ? ui.c.green(text) : ui.c.yellow(text);
+}
+
 function printApp(app, { verbose }) {
   console.log('');
   console.log(`  ${ui.c.bold(app.appName)} ${ui.c.dim(app.appKey)}  ${modeLabel(app)}`);
   const summary = app.summary || {};
+  const quality = summary.quality || {};
   ui.keyValue([
     ['Rules', `${summary.active || 0} active of ${summary.rules || 0} (${summary.blockReady || 0} can block, ${summary.blocking || 0} blocking)`],
     ['Alerts, last 7 days', `${summary.alerts7d || 0} (${summary.falsePositives7d || 0} marked false positive)`],
+    ['Precision, last 14 days', quality.precision == null
+      ? ui.c.dim('not measured yet: no labeled alerts')
+      : `${precisionCell(quality)} right${quality.wrong ? `, ${quality.wrong} wrong` : ''}${quality.unclear ? `, ${quality.unclear} unclear` : ''}`],
     ['Pack version', `${app.packVersion || '-'} (latest ${app.latestPackVersion})${summary.customized ? `, ${summary.customized} rule(s) edited by a person` : ''}${summary.missing ? `, ${summary.missing} missing` : ''}`],
   ]);
   if (app.mode === 'monitor') {
@@ -50,16 +64,18 @@ function printApp(app, { verbose }) {
   }
   if (verbose) {
     ui.table(
-      ['Rule', 'Kind', 'Mode', 'Status', 'Alerts 7d', 'Blocks'],
+      ['Rule', 'Kind', 'Mode', 'Status', 'Alerts 7d', 'Precision 14d', 'Blocks'],
       (app.rules || []).map((rule) => [
         ui.truncate(rule.name, 52),
         rule.kind,
         rule.present ? rule.mode : '-',
         rule.present ? `${rule.status}${rule.customized ? ' (edited)' : ''}` : ui.c.dim('missing'),
         String(rule.alerts7d || 0),
+        rule.present ? precisionCell(rule.quality) : '-',
         rule.blocking ? ui.c.green('yes') : rule.blockReady ? ui.c.dim('ready') : ui.c.dim('alerts only'),
       ])
     );
+    console.log(`  ${ui.c.dim('A rule\'s labeled alerts and what detection learning changed: securenow alerts rules quality <rule-id> (ids: --json)')}`);
   }
 }
 
@@ -134,4 +150,50 @@ async function monitor(args, flags) {
   return change('/protection/monitor', flags, { verb: 'Switching to monitor mode for' });
 }
 
-module.exports = { status, enable, disable, block, monitor };
+const LEARNED = {
+  exclusion_added: (item) => `${item.ruleKey} on ${item.appKey}: learned an exclusion for legitimate traffic; precision on the same alerts ${item.after && item.after.precision != null ? `${(item.after.precision * 100).toFixed(1)}%` : 'recomputed next run'}`,
+  needs_person: (item) => `${item.ruleKey} on ${item.appKey}: below ${PRECISION_TARGET * 100}% and no exclusion passed the safety checks; a person should tune it${item.refusals && item.refusals[0] ? ` (${ui.truncate(item.refusals[0], 120)})` : ''}`,
+};
+
+// Detection learning now (it also runs every hour): label the starter rules' alerts (was the rule right?), measure
+// each rule's precision, and learn exclusions for legitimate traffic that pass the safety checks.
+async function learn(args, flags) {
+  requireAuth();
+  const appKey = flags.app ? String(flags.app) : (flags.all ? null : resolveAppKey(flags));
+  const s = ui.spinner(appKey ? `Running detection learning for ${appKey}` : 'Running detection learning');
+  let data;
+  try {
+    data = await api.post('/protection/learn', appKey ? { app: appKey } : {});
+    s.stop(data.message || 'Done');
+  } catch (err) {
+    s.fail('Detection learning failed');
+    throw err;
+  }
+  if (flags.json) { ui.json(data); return; }
+  const rules = (data.quality || []).filter((rule) => rule.labeled || rule.unclear || rule.unlabeled || rule.superseded);
+  console.log('');
+  if (rules.length === 0) {
+    ui.info('No starter-rule alerts in the last 14 days, so there is nothing to measure yet.');
+  } else {
+    ui.table(
+      ['Rule', 'App', 'Precision', 'Right', 'Wrong', 'Unclear', 'Not labeled', 'Fixed'],
+      rules.map((rule) => [
+        rule.ruleKey,
+        ui.truncate(rule.appKey || '-', 12),
+        precisionCell(rule),
+        String(rule.correct || 0),
+        String(rule.wrong || 0),
+        String(rule.unclear || 0),
+        String(rule.unlabeled || 0),
+        String(rule.fixedWrong || 0),
+      ])
+    );
+    console.log(`  ${ui.c.dim('Precision counts the alerts the rule as it is now would still raise; Fixed = wrong alerts a rule change or exclusion no longer raises.')}`);
+  }
+  for (const item of data.learned || []) {
+    console.log(`  ${item.kind === 'exclusion_added' ? ui.c.green('learned') : ui.c.yellow('needs you')}  ${(LEARNED[item.kind] || ((x) => `${x.ruleKey}: ${x.kind}`))(item)}`);
+  }
+  console.log('');
+}
+
+module.exports = { status, enable, disable, block, monitor, learn };

package/cli/security.js

@@ -106,6 +106,9 @@ async function alertRulesRoute(args, flags) {
   if (sub === 'why' || sub === 'explain') {
     return alertRuleWhy(args.slice(1), flags);
   }
+  if (sub === 'quality' || sub === 'precision') {
+    return alertRuleQuality(args.slice(1), flags);
+  }
   if (sub === 'promote') {
     return alertRuleSetMode(args.slice(1), flags, 'prod');
   }
@@ -1322,6 +1325,75 @@ async function alertRuleWhy(args, flags) {
   console.log('');
 }
 
+// Was the rule right to fire? Its precision over the last 14 days from the labeled alerts, the ones labeled wrong
+// or unclear and why, and what detection learning changed.
+async function alertRuleQuality(args, flags) {
+  requireAuth();
+  const id = args[0];
+  if (!id) {
+    ui.error('Usage: securenow alerts rules quality <rule-id> [--json]');
+    process.exit(1);
+  }
+  const s = ui.spinner('Reading the rule\'s labeled alerts');
+  let data;
+  try {
+    data = await api.get(`/alert-rules/${id}/quality`);
+    s.stop('');
+  } catch (err) {
+    s.fail('Failed to read the rule\'s quality');
+    throw err;
+  }
+  if (flags.json) { ui.json(data); return; }
+  const q = data.quality || {};
+  const target = data.target || 0.95;
+  const pct = (value) => `${(value * 100).toFixed(value === 1 ? 0 : 1)}%`;
+  console.log('');
+  ui.heading(data.rule?.name || 'Rule quality');
+  ui.keyValue([
+    [`Precision, last ${data.windowDays || 14} days`, q.precision == null
+      ? ui.c.dim('not measured yet: no alert labeled right or wrong')
+      : (q.precision >= target ? ui.c.green : ui.c.yellow)(`${pct(q.precision)} (target ${pct(target)})`)],
+    ['Right', `${q.correct || 0} (${q.actionable || 0} worth acting on)`],
+    ['Wrong', String(q.wrong || 0)],
+    ['Unclear', String(q.unclear || 0)],
+    ['Not labeled yet', String(q.unlabeled || 0)],
+    ['No longer raised', `${q.superseded || 0}: ${q.fixedWrong || 0} wrong alert(s) fixed by a rule change or exclusion, ${q.droppedCorrect || 0} correct one(s) dropped`],
+    ['Never reached anyone', `${q.dismissed || 0} (closed by an exclusion or the pre-filter)`],
+  ]);
+  const rowsTable = (title, rows) => {
+    if (!rows || rows.length === 0) return;
+    console.log('');
+    console.log(`  ${ui.c.bold(title)}`);
+    ui.table(['When (UTC)', 'Address', 'Request', 'Answer', 'By', 'Why'], rows.map((row) => [
+      new Date(row.at).toISOString().replace('T', ' ').slice(0, 16),
+      row.ip,
+      ui.truncate((row.paths || [])[0] || '-', 40),
+      (row.answers || []).join(',') || '-',
+      row.labeledBy || '-',
+      ui.truncate(row.benignPattern || row.reason || '', 60),
+    ]));
+  };
+  rowsTable('Labeled wrong', data.wrong);
+  rowsTable('Unclear', data.unclear);
+  rowsTable('Wrong alerts the rule no longer raises (fixed)', data.fixed);
+  rowsTable('Correct alerts the rule no longer raises (check the change didn\'t lose real attacks)', data.dropped);
+  if ((data.learning || []).length) {
+    console.log('');
+    console.log(`  ${ui.c.bold('Detection learning')}`);
+    for (const action of data.learning) {
+      const when = new Date(action.at).toISOString().slice(0, 10);
+      if (action.kind === 'exclusion_added') {
+        console.log(`  ${when}  ${ui.c.green('exclusion learned')}  ${pct(action.before?.precision ?? 0)} → ${pct(action.after?.precision ?? 0)} on the same alerts${action.exclusionId ? ui.c.dim(`  (exclusion ${action.exclusionId})`) : ''}`);
+      } else {
+        console.log(`  ${when}  ${ui.c.yellow('needs a person')}  ${ui.truncate(action.reason || '', 110)}`);
+      }
+    }
+  }
+  console.log('');
+  console.log(ui.c.dim(`  Exclusions on this rule: securenow alerts rules exclusions ${id} list`));
+  console.log('');
+}
+
 // An agent hands the promotion of its test-mode rule to a person: the request waits on the rules
 // page and in `securenow human promotions` until someone approves or denies it.
 async function alertRuleRequestPromotion(args, flags) {

package/cli.js

@@ -359,12 +359,13 @@ const COMMANDS = {
     sub: {
       rules: {
         desc: 'Create (from English or SQL), list, show, update, duplicate, promote/demote, set-sql, validate, delete, test, run, explain, or tune alert rules',
-        usage: 'securenow alerts rules <list|from-text|create|show|update|duplicate|promote|demote|set-sql|validate|delete|test|run|runs|why|dry-run-query|tune-query|exclusions|versions|version|rollback> [options]\n'
+        usage: 'securenow alerts rules <list|from-text|create|show|update|duplicate|promote|demote|set-sql|validate|delete|test|run|runs|why|quality|dry-run-query|tune-query|exclusions|versions|version|rollback> [options]\n'
           + '  securenow alerts rules from-text "<what to catch, in English>" --app <key> [--env <environment>] [--instant|--periodic] [--yes|--draft-only]\n'
           + '  securenow alerts rules change <id> "<what should change>" [--yes|--draft-only]   rewrite a rule from English, checked, applied as a new version\n'
           + '  securenow alerts rules runs <id> [--limit 20] [--since 1h]     what each run saw and why it did or didn\'t notify\n'
           + '  securenow alerts rules run <id> [--no-wait]                     run a periodic rule now\n'
           + '  securenow alerts rules why <id> --ip <ip> [--since 1h]          why the rule did or didn\'t alert on one address\n'
+          + '  securenow alerts rules quality <id>                             was the rule right? precision, wrong alerts, what was learned\n'
           + '  securenow alerts rules request-promotion <id> --reason "..."    ask a person to promote a test-mode rule (for agents)',
         flags: {
           json: 'Output as JSON',
@@ -458,14 +459,15 @@ const COMMANDS = {
   },
   protect: {
     desc: 'Starter protection per app: the rules every app gets, monitor or block mode',
-    usage: 'securenow protect <status|enable|disable|block|monitor> [--app <key>] [--json]',
-    flags: { app: 'App key (defaults to logged-in app)', all: 'With status: every app', rules: 'With status: list each rule', yes: 'Skip the confirmation (block, disable)', json: 'Output as JSON' },
+    usage: 'securenow protect <status|enable|disable|block|monitor|learn> [--app <key>] [--json]',
+    flags: { app: 'App key (defaults to logged-in app)', all: 'With status or learn: every app', rules: 'With status: list each rule', yes: 'Skip the confirmation (block, disable)', json: 'Output as JSON' },
     sub: {
       status: { desc: 'Each app\'s starter protection: mode, rules, alerts and whether it is ready to block', usage: 'securenow protect status [--app <key> | --all] [--rules] [--json]', run: (a, f) => require('./cli/protect').status(a, f) },
       enable: { desc: 'Turn starter protection on for an app (monitor mode: rules alert, nothing is blocked)', usage: 'securenow protect enable [--app <key>]', run: (a, f) => require('./cli/protect').enable(a, f) },
       disable: { desc: 'Turn starter protection off for an app; it stays off until someone turns it on', usage: 'securenow protect disable [--app <key>] [--yes]', run: (a, f) => require('./cli/protect').disable(a, f) },
       block: { desc: 'Turn blocking on (a person signed in): block-ready rules block exploit signatures for 6 hours', usage: 'securenow protect block [--app <key>] [--yes]', run: (a, f) => require('./cli/protect').block(a, f) },
       monitor: { desc: 'Back to monitor mode: the starter rules alert, nothing is blocked by them', usage: 'securenow protect monitor [--app <key>]', run: (a, f) => require('./cli/protect').monitor(a, f) },
+      learn: { desc: 'Detection learning now (it also runs hourly): label the starter rules\' alerts, measure precision, learn safe exclusions', usage: 'securenow protect learn [--app <key> | --all] [--json]', run: (a, f) => require('./cli/protect').learn(a, f) },
     },
     defaultSub: 'status',
   },

package/mcp/catalog.js

@@ -1345,6 +1345,34 @@ const TOOLS = [
       ...confirmSchema,
     }, ['app', 'confirm', 'reason']),
   },
+  {
+    name: 'securenow_alert_rule_quality',
+    title: 'Alert Rule Quality (Was It Right to Fire?)',
+    description: 'How often an alert rule is right, from its labeled alerts of the last 14 days: correct (an attack, probe or scan, even failed or small), wrong (the rule mistook legitimate use of the app for an attack), unclear; precision = correct / (correct + wrong) over the alerts the rule as it is now would still raise (each alert is replayed through the current conditions and exclusions), target 95%. fixed lists wrong alerts a rule change or exclusion no longer raises; dropped lists correct ones it no longer raises (lost detections). Lists the wrong and unclear alerts with the reason and the legitimate pattern, a sample of correct ones, and what detection learning did: an exclusion it learned for legitimate traffic (with before/after precision) or a "needs a person" analysis when no safe fix passed its safety gate.',
+    scope: 'alerts:read',
+    readOnly: true,
+    method: 'GET',
+    endpoint: '/alert-rules/:id/quality',
+    pathParams: ['id'],
+    inputSchema: objectSchema({
+      id: string('Alert rule id.'),
+    }, ['id']),
+  },
+  {
+    name: 'securenow_protection_learn',
+    title: 'Run Detection Learning Now',
+    description: 'Run detection learning for the account now (it also runs every hour): label the base rules\' recent alerts (was the rule right?), measure each rule\'s precision per app, and learn a safe exclusion for a rule below 95% that keeps mistaking an app\'s legitimate traffic for an attack. A learned exclusion never names an IP and must keep every correct alert and every known attack firing. An account where detection learning isn\'t on yet gets 409 not_rolled_out. Write action; requires confirmation.',
+    scope: 'alerts:write',
+    readOnly: false,
+    confirm: true,
+    method: 'POST',
+    endpoint: '/protection/learn',
+    bodyFields: ['app'],
+    inputSchema: objectSchema({
+      app: string('Optional application key; every app when left out.'),
+      ...confirmSchema,
+    }, ['confirm', 'reason']),
+  },
   {
     name: 'securenow_protection_disable',
     title: 'Turn Starter Protection Off',
@@ -2699,7 +2727,7 @@ const AGENT_PATROL_RUNBOOK = [
   'B. Requires Human: securenow_human_actions_grouped_list, largest groups first (rows you already decided are hidden). Each group lists up to 12 rows; verify and decide 2 to 4 rows of a group, then move to the next group. For each IP row:',
   '   - verify the IP;',
   '   - verdict malicious and blockAllowed: securenow_agent_block_ip with ip, verificationId, notificationId, a reason naming the 2 or 3 decisive signals, confirm true;',
-  '   - verdict benign or never_block: when the rule plainly misfired on this kind of legitimate traffic, securenow_human_action_false_positive with the verificationId, on this rule only (no ruleScope, no applyToExisting). On a test-mode rule that will keep misfiring on this IP, add createExclusion with at most a path condition: the server pins the exclusion to this IP and it expires within 7 days. On a prod rule, never createExclusion. Otherwise securenow_human_action_close_no_write with outcome "skipped" (or "other"), the verificationId and a one-line summary of the evidence. An IP the org has blocked stays blocked: lifting a block is for a person;',
+  '   - verdict benign or never_block: a false positive means the rule mistook LEGITIMATE use of the app for an attack. A real probe, scan or attack attempt that is just too small to block (one failed request for /.env, a scanner fetching a page) is not a false positive: the rule was right, so close it with securenow_human_action_close_no_write, outcome "skipped". When the rule plainly misfired on legitimate traffic, securenow_human_action_false_positive with the verificationId, on this rule only (no ruleScope, no applyToExisting). On a test-mode rule that will keep misfiring on this IP, add createExclusion with at most a path condition: the server pins the exclusion to this IP and it expires within 7 days. On a prod rule, never createExclusion. Otherwise securenow_human_action_close_no_write with outcome "skipped" (or "other"), the verificationId and a one-line summary of the evidence. An IP the org has blocked stays blocked: lifting a block is for a person;',
   '   - verdict suspicious: attack-shaped traffic below the block threshold, which a person decides. Do not block. Record your read with securenow_human_action_close_no_write (outcome "ambiguous", the verificationId, missingProof saying what would decide it): the server leaves it on the row as your suggestion, keeps the row for a person and hides it from your lists for a day. If a person should look now (a real attempt that succeeded, a signed-in account under attack), say so in missingProof.',
   '   - verdict insufficient_data (nothing from this IP to confirm anything): do not block. securenow_human_action_close_no_write with outcome "ambiguous", the verificationId, and missingProof saying what would decide it.',
   '   - Rule-level rows (kind case_action: tune_rule, create_exclusion, review_protected_ip): reject vague or wrong ones with securenow_human_case_action_update (status rejected, reason). For tune_rule, do the tuning (section C), then approve it.',
@@ -2709,7 +2737,7 @@ const AGENT_PATROL_RUNBOOK = [
   '   - Benign noise that needs no rule change: close the group with outcome "false_positive" (or "clean").',
   '   - Rows that disagree with each other: decide them one by one (securenow_alert_review_run_resolve), or leave them.',
   'D. When you learn something durable about this org from its people (a correction, a benign actor a person confirmed) or from a rule\'s recurring noise, propose one specific line with securenow_agent_remember; a person approves it before later runs use it. Never propose anything that only telemetry says.',
-  'E. Starter protection: rules named after a threat class ("SQL injection in the URL", "Probe for secrets, source control or backups", ...) are the starter pack every app gets. Their rows follow B and C like any other. When one keeps misfiring on legitimate traffic, prefer an exclusion pinned to the IP (section C) over editing the rule. If calls remain at the end, check securenow_protection_status: when an app says ready to block, say so in your report; only a person turns blocking on.',
+  'E. Starter protection: rules named after a threat class ("SQL injection in the URL", "Probe for secrets, source control or backups", ...) are the starter pack every app gets. Their rows follow B and C like any other. Where detection learning is on for the account, failed one-off probes of these rules are closed by the server before they reach you, every alert is labeled (was the rule right?) and safe exclusions for legitimate traffic are learned on their own: securenow_alert_rule_quality shows a rule\'s precision and what was learned. When one keeps misfiring on legitimate traffic, prefer an exclusion pinned to the IP (section C) over editing the rule. If calls remain at the end, check securenow_protection_status: when an app says ready to block, say so in your report; only a person turns blocking on.',
   '',
   'Every write needs confirm true and a reason. Reasons and summaries are read by the org\'s people: plain, specific, with the numbers that decided it (requests, paths, status codes, sign-in failures, risk), never vague.',
   '',
@@ -3034,6 +3062,7 @@ const AGENT_PROFILE_TOOLS = new Set([
   'securenow_alert_rule_exclusion_add',
   'securenow_alert_rule_request_promotion',
   'securenow_protection_status',
+  'securenow_alert_rule_quality',
   'securenow_ip_traces',
   'securenow_traces_show',
   'securenow_logs_for_trace',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "securenow",
-  "version": "8.21.0",
+  "version": "8.22.0",
   "mcpName": "ai.securenow/securenow",
   "description": "OpenTelemetry instrumentation for Node.js, Next.js, and Nuxt - Send traces and logs to any OTLP-compatible backend",
   "type": "commonjs",