sift-light 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/README.md +11 -4
  3. package/README.zh-CN.md +11 -4
  4. package/package.json +1 -1
  5. package/plugins/sift-light/.claude-plugin/plugin.json +1 -1
  6. package/plugins/sift-light/.codex-plugin/plugin.json +1 -1
  7. package/plugins/sift-light/.mcp.json +1 -1
  8. package/plugins/sift-light/concept-worker.mjs +39 -12
  9. package/plugins/sift-light/kimi.plugin.json +2 -2
  10. package/plugins/sift-light/omp-extension.mjs +558 -83
  11. package/plugins/sift-light/package.json +1 -1
  12. package/src/analysis-limits.ts +1 -1
  13. package/src/analysis-store.ts +6 -2
  14. package/src/analysis-types.ts +10 -1
  15. package/src/concept-inference.ts +44 -11
  16. package/src/concept-model.ts +2 -2
  17. package/src/concept-search.ts +12 -2
  18. package/src/concept-worker.mjs +39 -12
  19. package/src/concept-worker.ts +1 -1
  20. package/src/config-reader.ts +11 -3
  21. package/src/evidence-service.ts +117 -13
  22. package/src/file-discovery.ts +111 -17
  23. package/src/hybrid-search.ts +13 -0
  24. package/src/index.ts +4 -10
  25. package/src/inspect.ts +3 -0
  26. package/src/language-capability-definitions.ts +9 -0
  27. package/src/mcp-model-output.ts +47 -1
  28. package/src/mcp-semantic-judge.ts +19 -8
  29. package/src/mcp-server.mjs +754 -247
  30. package/src/mcp-server.ts +11 -13
  31. package/src/mcp.ts +2 -0
  32. package/src/model-error.ts +6 -16
  33. package/src/omp-index.ts +4 -6
  34. package/src/prompt-guidelines.ts +5 -5
  35. package/src/python-roles.ts +247 -0
  36. package/src/redaction.ts +9 -2
  37. package/src/request-aliases.ts +48 -0
  38. package/src/request-contract-catalog.ts +24 -7
  39. package/src/request-contract.ts +60 -6
  40. package/src/request.ts +3 -1
  41. package/src/runtime.ts +2 -2
  42. package/src/semantic-judge.ts +17 -3
  43. package/src/service.ts +31 -1
  44. package/src/session-summary.ts +4 -4
  45. package/src/source-inspection.ts +27 -2
  46. package/src/tool-schema.ts +18 -18
  47. package/src/tui/layout.ts +2 -2
  48. package/src/tui/renderers.ts +2 -2
  49. package/src/types.ts +4 -0
package/src/mcp-server.ts CHANGED
@@ -8,10 +8,7 @@ import {
8
8
  createDefaultSiftLightMcpService,
9
9
  startSiftLightMcpServer,
10
10
  } from "./mcp.js";
11
- import {
12
- createMcpSemanticJudgeIntegration,
13
- mcpSemanticJudgeConfigSource,
14
- } from "./mcp-semantic-judge.js";
11
+ import { createMcpSearchFeatures, mcpSemanticJudgeConfigSource } from "./mcp-semantic-judge.js";
15
12
  import type { SemanticJudgeIntegration } from "./semantic-judge.js";
16
13
  import { parseSiftLightMcpTransport, SIFT_LIGHT_MCP_USAGE } from "./mcp-cli.js";
17
14
  import { parseSiftLightMcpOutputMode, type SiftLightMcpOutputMode } from "./mcp-output.js";
@@ -41,10 +38,6 @@ function allowedOrigins(): string[] {
41
38
  .filter((origin) => origin.length > 0);
42
39
  }
43
40
 
44
- async function configuredSemanticJudge(): Promise<SemanticJudgeIntegration> {
45
- return createMcpSemanticJudgeIntegration();
46
- }
47
-
48
41
  function logSemanticJudgeStatus(integration: SemanticJudgeIntegration): void {
49
42
  const status = integration.config.enabled ? "enabled" : "disabled";
50
43
  process.stderr.write(
@@ -55,12 +48,13 @@ function logSemanticJudgeStatus(integration: SemanticJudgeIntegration): void {
55
48
  async function runHttpServer(
56
49
  outputMode: SiftLightMcpOutputMode,
57
50
  semanticJudge: SemanticJudgeIntegration | undefined,
51
+ vectorSearchEnabled: boolean,
58
52
  ): Promise<void> {
59
53
  const running = await startSiftLightMcpServer({
60
54
  cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.cwd(),
61
55
  host: process.env.SIFT_LIGHT_MCP_HOST ?? DEFAULT_MCP_HOST,
62
56
  port: environmentInteger("SIFT_LIGHT_MCP_PORT", DEFAULT_MCP_PORT, 0, 65_535),
63
- createService: () => createDefaultSiftLightMcpService(semanticJudge),
57
+ createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled),
64
58
  maxSessions: environmentInteger(
65
59
  "SIFT_LIGHT_MCP_MAX_SESSIONS",
66
60
  DEFAULT_MCP_MAX_SESSIONS,
@@ -108,11 +102,12 @@ async function runHttpServer(
108
102
  async function runStdioServer(
109
103
  outputMode: SiftLightMcpOutputMode,
110
104
  semanticJudge: SemanticJudgeIntegration | undefined,
105
+ vectorSearchEnabled: boolean,
111
106
  ): Promise<void> {
112
107
  const running = await startSiftLightMcpStdioServer({
113
108
  cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.env.CLAUDE_PROJECT_DIR ?? process.cwd(),
114
109
  outputMode,
115
- createService: () => createDefaultSiftLightMcpService(semanticJudge),
110
+ createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled),
116
111
  });
117
112
  process.stderr.write("sift-light MCP serving one local client over stdio\n");
118
113
  process.stderr.write(`sift-light MCP working directory: ${running.cwd}\n`);
@@ -140,13 +135,16 @@ async function main(): Promise<void> {
140
135
  return;
141
136
  }
142
137
  const outputMode = parseSiftLightMcpOutputMode(process.env.SIFT_LIGHT_MCP_OUTPUT_MODE);
143
- const semanticJudge = await configuredSemanticJudge();
138
+ const { semanticJudge, vectorSearchEnabled } = await createMcpSearchFeatures();
144
139
  logSemanticJudgeStatus(semanticJudge);
140
+ process.stderr.write(
141
+ `sift-light MCP vector search: ${vectorSearchEnabled ? "enabled" : "disabled"}\n`,
142
+ );
145
143
  if (transport === "stdio") {
146
- await runStdioServer(outputMode, semanticJudge);
144
+ await runStdioServer(outputMode, semanticJudge, vectorSearchEnabled);
147
145
  return;
148
146
  }
149
- await runHttpServer(outputMode, semanticJudge);
147
+ await runHttpServer(outputMode, semanticJudge, vectorSearchEnabled);
150
148
  }
151
149
 
152
150
  try {
package/src/mcp.ts CHANGED
@@ -93,12 +93,14 @@ export interface SiftLightMcpService {
93
93
 
94
94
  export function createDefaultSiftLightMcpService(
95
95
  semanticJudge?: SemanticJudgeIntegration,
96
+ vectorSearchEnabled = false,
96
97
  ): SiftLightMcpService {
97
98
  const resolvedSemanticJudge =
98
99
  semanticJudge ?? createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG);
99
100
  return new SiftLightService({
100
101
  runRipgrep: createRipgrepRunner(),
101
102
  structure: createCtagsStructureProvider(),
103
+ vectorSearchEnabled,
102
104
  semanticJudge: resolvedSemanticJudge,
103
105
  });
104
106
  }
@@ -73,19 +73,14 @@ export interface RequestContractProjection {
73
73
  text: string;
74
74
  }
75
75
 
76
- function projectRequestContract(
77
- projected: RequestContractDetails,
78
- serialized: string,
79
- message: string,
80
- ): string {
81
- const prefix = `sift-light failed: request rejected [${projected.code}]: ${message}`;
76
+ function projectRequestContract(projected: RequestContractDetails, serialized: string): string {
82
77
  const recovery = projected.recovery;
83
78
  const next = recovery.nextRequest
84
79
  ? "\nCopy the nested recovery.nextRequest object unchanged; do not repeat the original query."
85
- : `\nRecovery action: ${recovery.action}. ${recovery.reason}`;
80
+ : `\nRecovery action: ${recovery.action}.`;
86
81
  // Put the machine payload first so simple host adapters that locate the
87
82
  // first `nextRequest` marker see the exact nested request object.
88
- return `\nError details: ${serialized}\n${prefix}${next}`;
83
+ return `\nsift-light failed: request rejected [${projected.code}].\nError details: ${serialized}${next}`;
89
84
  }
90
85
 
91
86
  /** Build the one bounded contract projection consumed by structured and text hosts. */
@@ -94,11 +89,10 @@ export function requestContractProjection(
94
89
  ): RequestContractProjection {
95
90
  const details = boundedRequestContractDetails(error.details);
96
91
  const serializedDetails = JSON.stringify(details);
97
- const boundedMessage = error.message.toWellFormed().slice(0, 1_024);
98
- const result = projectRequestContract(details, serializedDetails, boundedMessage);
92
+ const result = projectRequestContract(details, serializedDetails);
99
93
  if (Buffer.byteLength(result) <= MAX_REQUEST_RECOVERY_BYTES) return { details, text: result };
100
94
  const compact: RequestContractDetails = {
101
- code: boundedMessage.length > 0 ? details.code : "E_REQUEST_CONTRACT_PAYLOAD",
95
+ code: details.code,
102
96
  ...(details.mode ? { mode: details.mode } : {}),
103
97
  issues: [
104
98
  {
@@ -111,10 +105,6 @@ export function requestContractProjection(
111
105
  reason: "Exact recovery was omitted; correct the request explicitly.",
112
106
  },
113
107
  };
114
- const compactText = projectRequestContract(
115
- compact,
116
- JSON.stringify(compact),
117
- "The request-contract error exceeded the bounded payload budget; correct the request explicitly.",
118
- );
108
+ const compactText = projectRequestContract(compact, JSON.stringify(compact));
119
109
  return { details: compact, text: compactText };
120
110
  }
package/src/omp-index.ts CHANGED
@@ -1,7 +1,6 @@
1
1
  import { homedir } from "node:os";
2
2
  import { join } from "node:path";
3
3
  import {
4
- DEFAULT_SEMANTIC_JUDGE_CONFIG,
5
4
  normalizeSearchEnforcement,
6
5
  readSiftLightConfigFile,
7
6
  resolveSiftLightConfigPath,
@@ -27,7 +26,7 @@ import {
27
26
  } from "./search-policy.js";
28
27
  import { siftLightSchema } from "./tool-schema.js";
29
28
  import { modelErrorText } from "./model-error.js";
30
- import { createSemanticJudgeIntegration } from "./semantic-judge.js";
29
+ import { createConfiguredSemanticJudgeIntegration } from "./semantic-judge.js";
31
30
 
32
31
  const SIFT_LIGHT_LABEL = "sift-light";
33
32
  const OMP_REPLACED_SEARCH_TOOLS = new Set(["grep", "glob"]);
@@ -188,13 +187,12 @@ export async function registerOmpSiftLightExtension(
188
187
  (await readSiftLightConfigFile(resolveSiftLightConfigPath(ompAgentDir()), {
189
188
  missing: process.env[SIFT_LIGHT_CONFIG_ENV]?.trim() ? "error" : "defaults",
190
189
  }));
191
- const semanticJudge = createSemanticJudgeIntegration(
192
- resolvedConfig.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG,
193
- );
190
+ const semanticJudge = createConfiguredSemanticJudgeIntegration(resolvedConfig);
194
191
  const runtime = new SiftLightRuntime(
195
192
  new SiftLightService({
196
193
  runRipgrep: createRipgrepRunner(),
197
194
  structure: createCtagsStructureProvider(),
195
+ vectorSearchEnabled: resolvedConfig.vectorSearchEnabled === true,
198
196
  semanticJudge,
199
197
  }),
200
198
  );
@@ -218,7 +216,7 @@ export async function registerOmpSiftLightExtension(
218
216
  name: SIFT_LIGHT_LABEL,
219
217
  label: SIFT_LIGHT_LABEL,
220
218
  description:
221
- "Search and navigate code with bounded, verifiable evidence. Use pattern for content or mode=files with query for filenames.",
219
+ "Search code with bounded evidence. Routine searches use pattern for fast exact content or mode=files with query for filenames; concept/hybrid require vectorSearchEnabled:true in sift-light.json.",
222
220
  approval: "read",
223
221
  promptSnippet: "Search file contents without flooding context",
224
222
  promptGuidelines: siftLightPromptGuidelines(),
@@ -5,7 +5,7 @@ const SOURCE_OUTPUT_GUIDANCE =
5
5
 
6
6
  export function siftLightPromptGuidelines(structuredOutput = true): string[] {
7
7
  return [
8
- `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`,
8
+ `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} For routine development searches, start with fast exact content, filename or applicable structural modes when the request has a usable name, symbol, error text or other literal clue. Omitted mode is ordinary exact search and never loads the local embedding model. Vector search is disabled by default; concept/hybrid require vectorSearchEnabled:true in sift-light.json and an installed model. They can take tens of seconds on an uncached scope, so select them only when semantic recall is needed. Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`,
9
9
  `An omitted path searches the project cwd. Use scope:"strict" for a question restricted to one path; otherwise, if an explicit subpath has zero matches, ordinary and content-analysis searches retry from cwd and return project-wide counts with an expansion notice. Explicit absolute paths and .. traversal can search outside cwd, except protected external system areas and .git internals. Git changes mode remains cwd-scoped.`,
10
10
  `Search output includes counts, categories, ranked paths, coverage and continuation metadata. Source excerpts may contain the searched text. Use mode="inspect" or the host read capability when exact source is required for an edit or verification.`,
11
11
  `Use file and directory distributions to choose evidence. Reuse the visible cursor with path or paths for match metadata; mode="summary" pages the remaining file statistics. Match counts are not relevance scores.`,
@@ -14,7 +14,7 @@ export function siftLightPromptGuidelines(structuredOutput = true): string[] {
14
14
  `Use anyOf:["term1","term2"] when every exact occurrence of 2-64 literals is needed in one version-bound result. It is case-sensitive, reports anonymized condition counts, and runs requests above eight terms as bounded parallel chunks. Large condition inventories have separate continuation pages; copy those requests to retrieve the complete counts.`,
15
15
  `For a changed-code question, add changes:{base:"HEAD",scope:"lines",side:"new"}; omit target for the working tree, use side:"old" for deleted-side statistics. Copy returned continuation requests to preserve source versions.`,
16
16
  `Use mode:"capabilities" when the language or requested operation is unclear to get a compact lazy inventory. Use mode:"outline" with a concrete source file path for symbol counts and locations, mode:"imports" for static relationships, and mode:"tests" for related-test candidates. Their text pages summarize metadata; structured details can retain source evidence.`,
17
- `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; business concepts belong in hybrid/concept. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`,
17
+ `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; use hybrid/concept only for business concepts that cannot be located by a literal clue. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`,
18
18
  `Use mode:"structure" only for JS/TS/TSX/Go, with a required nonempty ast-grep pattern such as "compare($X, $X)" or "send()" for code shapes across whitespace; the text page reports structural counts and locations; structured details can retain matched source evidence.`,
19
19
  `Use mode:"concept" plus a natural-language query when names are unknown. It runs a pinned local multilingual model only after explicit installation; no search downloads weights or sends code to a remote model. Results expose candidate counts, score statistics, paths and the bounded ranked passage behind each candidate. Similarity scores identify candidates, not correctness.`,
20
20
  `Use mode:"hybrid" plus query when wording may differ from the source. It reports exact and semantic counts separately, removes overlap, and retains one pageable evidence snapshot. conceptLimit changes only the semantic candidate count; retained candidates keep their bounded source passages.`,
@@ -28,9 +28,9 @@ export function siftLightPromptGuidelines(structuredOutput = true): string[] {
28
28
 
29
29
  function siftLightModelGuidelines(): string[] {
30
30
  return [
31
- `Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Omit mode/limit for automatic detail/summary selection. Compact model analysis pages may defer source excerpts to inspect.`,
32
- `Use ranked paths and condition counts to choose evidence. Cursor continuation pages the retained snapshot; summary file selection returns ordinary match metadata. Use mode="inspect" when an edit requires exact source.`,
33
- `Modes: capabilities for language prerequisites; files+query for filenames; anyOf/allOf for literal condition statistics; outline/imports/tests for structural and relationship summaries; structure for AST match statistics; concept/hybrid for semantic candidate statistics. Similarity and static links are not proof.`,
31
+ `Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Default search is exact and model-free. concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model; uncached runs may take tens of seconds. Omit mode/limit for auto pages.`,
32
+ `Use ranked paths and counts; Cursor continuation pages retained results. Use mode="inspect" for exact source before editing.`,
33
+ `Modes: files+query for filenames; anyOf/allOf for literals; outline/imports/tests for static code; structure for AST; concept/hybrid for semantic candidates. Similarity is not proof.`,
34
34
  `On rejection keep the strongest applicable mode and apply the stated repair once. Do not repeat the rejected request or include its error. Only explicit capability-unavailable permits a visibly partial alternative.`,
35
35
  ];
36
36
  }
@@ -0,0 +1,247 @@
1
+ import type { SourceDocument } from "./source-document.js";
2
+ import type { SyntaxAnalysis, SyntaxRole } from "./syntax-types.js";
3
+
4
+ /**
5
+ * Bounded lexical role scanner for Python.
6
+ *
7
+ * It separates comments, string literals and code exactly, and marks `def`/`class`
8
+ * names, import statements and call-shaped callees. It does not parse Python:
9
+ * comments, strings and code are lexical facts ("syntax" certainty); a call is a
10
+ * name followed by `(` in code and therefore stays a "candidate". Offsets are
11
+ * UTF-16 indices into `document.text`, like every other role provider.
12
+ */
13
+
14
+ const KEYWORDS = new Set([
15
+ "and",
16
+ "as",
17
+ "assert",
18
+ "async",
19
+ "await",
20
+ "case",
21
+ "del",
22
+ "elif",
23
+ "else",
24
+ "except",
25
+ "for",
26
+ "from",
27
+ "global",
28
+ "if",
29
+ "import",
30
+ "in",
31
+ "is",
32
+ "lambda",
33
+ "match",
34
+ "nonlocal",
35
+ "not",
36
+ "or",
37
+ "raise",
38
+ "return",
39
+ "while",
40
+ "with",
41
+ "yield",
42
+ ]);
43
+ const STRING_PREFIX = /^(?:[rRbBuUfF]|[rR][bBfF]|[bBfF][rR])?$/u;
44
+ const IDENTIFIER_START = /[\p{ID_Start}_]/u;
45
+ const IDENTIFIER_PART = /[\p{ID_Continue}]/u;
46
+
47
+ interface Span {
48
+ start: number;
49
+ end: number;
50
+ }
51
+
52
+ interface Lexed {
53
+ comments: Span[];
54
+ strings: Span[];
55
+ /** `true` at each UTF-16 index that belongs to code (outside comments/strings). */
56
+ code: Uint8Array;
57
+ }
58
+
59
+ function lex(text: string): Lexed {
60
+ const comments: Span[] = [];
61
+ const strings: Span[] = [];
62
+ const code = new Uint8Array(text.length).fill(1);
63
+ const mark = (start: number, end: number): void => {
64
+ code.fill(0, start, end);
65
+ };
66
+ let index = 0;
67
+ while (index < text.length) {
68
+ const character = text[index];
69
+ if (character === "#") {
70
+ const newline = text.indexOf("\n", index);
71
+ const end = newline < 0 ? text.length : newline;
72
+ comments.push({ start: index, end });
73
+ mark(index, end);
74
+ index = end;
75
+ continue;
76
+ }
77
+ if (character !== "'" && character !== '"') {
78
+ index += 1;
79
+ continue;
80
+ }
81
+ // A string prefix (r, b, f, u and their pairs) is part of the literal.
82
+ let prefixStart = index;
83
+ while (prefixStart > 0 && /[rRbBuUfF]/u.test(text[prefixStart - 1] ?? "")) prefixStart -= 1;
84
+ const prefix = text.slice(prefixStart, index);
85
+ const beforePrefix = text[prefixStart - 1] ?? "";
86
+ const start =
87
+ STRING_PREFIX.test(prefix) && !IDENTIFIER_PART.test(beforePrefix) ? prefixStart : index;
88
+ const triple = text.slice(index, index + 3) === character.repeat(3);
89
+ const delimiter = triple ? character.repeat(3) : character;
90
+ let cursor = index + delimiter.length;
91
+ let end = text.length;
92
+ while (cursor < text.length) {
93
+ const current = text[cursor];
94
+ // A backslash always protects the next character from ending the literal,
95
+ // including in raw strings (where the backslash itself is kept).
96
+ if (current === "\\") {
97
+ cursor += 2;
98
+ continue;
99
+ }
100
+ if (!triple && current === "\n") {
101
+ end = cursor;
102
+ break;
103
+ }
104
+ if (text.startsWith(delimiter, cursor)) {
105
+ end = cursor + delimiter.length;
106
+ break;
107
+ }
108
+ cursor += 1;
109
+ }
110
+ strings.push({ start, end });
111
+ mark(start, end);
112
+ index = end;
113
+ }
114
+ return { comments, strings, code };
115
+ }
116
+
117
+ function codeRanges(code: Uint8Array): Span[] {
118
+ const ranges: Span[] = [];
119
+ let start = -1;
120
+ for (let index = 0; index <= code.length; index += 1) {
121
+ const inCode = index < code.length && code[index] === 1;
122
+ if (inCode && start < 0) start = index;
123
+ else if (!inCode && start >= 0) {
124
+ ranges.push({ start, end: index });
125
+ start = -1;
126
+ }
127
+ }
128
+ return ranges;
129
+ }
130
+
131
+ function identifierEnd(text: string, start: number): number {
132
+ let end = start;
133
+ while (end < text.length && IDENTIFIER_PART.test(text[end] ?? "")) end += 1;
134
+ return end;
135
+ }
136
+
137
+ function skipSpaces(text: string, index: number, code: Uint8Array): number {
138
+ let cursor = index;
139
+ while (cursor < text.length && code[cursor] === 1 && /[ \t]/u.test(text[cursor] ?? ""))
140
+ cursor += 1;
141
+ return cursor;
142
+ }
143
+
144
+ /** Index after the statement starting at `start`, following parenthesized continuation lines. */
145
+ function statementEnd(text: string, start: number, code: Uint8Array): number {
146
+ let depth = 0;
147
+ for (let index = start; index < text.length; index += 1) {
148
+ if (code[index] !== 1) continue;
149
+ const character = text[index];
150
+ if (character === "(" || character === "[" || character === "{") depth += 1;
151
+ else if (character === ")" || character === "]" || character === "}")
152
+ depth = Math.max(0, depth - 1);
153
+ else if (character === "\\" && text[index + 1] === "\n") index += 1;
154
+ else if (character === "\n" && depth === 0) return index;
155
+ }
156
+ return text.length;
157
+ }
158
+
159
+ export function pythonRoleAnalysis(document: SourceDocument): SyntaxAnalysis {
160
+ const text = document.text;
161
+ const { comments, strings, code } = lex(text);
162
+ const roles: SyntaxRole[] = [];
163
+ const push = (
164
+ start: number,
165
+ end: number,
166
+ role: SyntaxRole["role"],
167
+ certainty: SyntaxRole["certainty"],
168
+ subkind?: string,
169
+ ): void => {
170
+ if (start < end)
171
+ roles.push({ start, end, role, certainty, node: 0, ...(subkind ? { subkind } : {}) });
172
+ };
173
+ for (const span of comments) push(span.start, span.end, "comment", "syntax");
174
+ for (const span of strings) push(span.start, span.end, "string", "syntax");
175
+ for (const span of codeRanges(code)) push(span.start, span.end, "code", "syntax");
176
+
177
+ let index = 0;
178
+ let lineStart = true;
179
+ while (index < text.length) {
180
+ const character = text[index] ?? "";
181
+ if (character === "\n") {
182
+ lineStart = true;
183
+ index += 1;
184
+ continue;
185
+ }
186
+ if (code[index] !== 1 || !IDENTIFIER_START.test(character)) {
187
+ if (!/[ \t]/u.test(character)) lineStart = false;
188
+ index += 1;
189
+ continue;
190
+ }
191
+ const previous = text[index - 1] ?? "";
192
+ if (IDENTIFIER_PART.test(previous)) {
193
+ index += 1;
194
+ continue;
195
+ }
196
+ const end = identifierEnd(text, index);
197
+ const word = text.slice(index, end);
198
+ const atStatementStart = lineStart;
199
+ lineStart = false;
200
+ if (atStatementStart && (word === "import" || word === "from")) {
201
+ const statement = statementEnd(text, index, code);
202
+ const body = text.slice(index, statement);
203
+ if (word === "import" || /\bimport\b/u.test(body)) {
204
+ push(index, statement, "import", "syntax", word === "from" ? "from-import" : "import");
205
+ index = statement;
206
+ continue;
207
+ }
208
+ }
209
+ if (word === "def" || word === "class") {
210
+ const nameStart = skipSpaces(text, end, code);
211
+ if (IDENTIFIER_START.test(text[nameStart] ?? "")) {
212
+ const nameEnd = identifierEnd(text, nameStart);
213
+ push(nameStart, nameEnd, "declaration", "syntax", word === "def" ? "function" : "class");
214
+ index = nameEnd;
215
+ continue;
216
+ }
217
+ }
218
+ // Attribute chains such as `client.fetch(` are one callee.
219
+ let chainEnd = end;
220
+ while (text[chainEnd] === "." && IDENTIFIER_START.test(text[chainEnd + 1] ?? "")) {
221
+ chainEnd = identifierEnd(text, chainEnd + 1);
222
+ }
223
+ const open = skipSpaces(text, chainEnd, code);
224
+ const lastSegment = text.slice(text.lastIndexOf(".", chainEnd - 1) + 1, chainEnd);
225
+ const before = text.slice(Math.max(0, index - 4), index);
226
+ if (
227
+ text[open] === "(" &&
228
+ code[open] === 1 &&
229
+ !KEYWORDS.has(word) &&
230
+ !KEYWORDS.has(lastSegment) &&
231
+ !/\bdef\s+$|\bclass\s+$/u.test(before)
232
+ ) {
233
+ push(index, open + 1, "call", "candidate", "lexical-call");
234
+ }
235
+ index = chainEnd;
236
+ }
237
+ roles.sort((a, b) => a.start - b.start || a.end - b.end || a.role.localeCompare(b.role));
238
+ return {
239
+ status: "ok",
240
+ nodes: [],
241
+ children: [],
242
+ symbols: [],
243
+ roles,
244
+ diagnostics: [],
245
+ limited: false,
246
+ };
247
+ }
package/src/redaction.ts CHANGED
@@ -7,6 +7,10 @@ const SENSITIVE_ASSIGNMENT = new RegExp(
7
7
  "gi",
8
8
  );
9
9
  const SENSITIVE_TOKEN = /\b(?:sk|ghp|xox[baprs])[-_][A-Za-z0-9][A-Za-z0-9_-]{7,}\b/g;
10
+ // An unquoted value that starts as a call or index expression (`env.get(`,
11
+ // `os.environ[`) is source code that reads a secret, not the secret itself.
12
+ // Masking it hides the evidence a reader needs, such as which key is read.
13
+ const CODE_EXPRESSION = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*[([]/u;
10
14
  const TYPE_ONLY_VALUES = new Set([
11
15
  "boolean",
12
16
  "number",
@@ -27,9 +31,12 @@ function redactString(value: string): { value: string; count: number } {
27
31
  SENSITIVE_ASSIGNMENT,
28
32
  (match: string, prefix: string, rawValue: string) => {
29
33
  const unquoted = rawValue.replace(/^["']|["']$/g, "").toLowerCase();
30
- if (TYPE_ONLY_VALUES.has(unquoted)) return match;
34
+ if (TYPE_ONLY_VALUES.has(unquoted) || CODE_EXPRESSION.test(rawValue)) return match;
31
35
  count += 1;
32
- return `${prefix}"[REDACTED]"`;
36
+ // Keep the source's own quoting: adding quotes to an unquoted value would
37
+ // misrepresent the file (for example, a bare `.env` value shown as quoted).
38
+ const quote = rawValue.startsWith('"') || rawValue.startsWith("'") ? rawValue[0] : "";
39
+ return `${prefix}${quote}[REDACTED]${quote}`;
33
40
  },
34
41
  );
35
42
  redacted = redacted.replace(SENSITIVE_TOKEN, () => {
@@ -0,0 +1,48 @@
1
+ import { SiftLightError } from "./errors.js";
2
+ import type { SiftLightInput } from "./service.js";
3
+
4
+ /**
5
+ * Field names that callers commonly send as a mode. They are not modes: the field
6
+ * alone selects the multi-term search. The schema accepts them so the request can
7
+ * be served, and the result discloses the normalization.
8
+ */
9
+ export const MODE_ALIASES = ["anyOf", "allOf"] as const;
10
+ export type ModeAlias = (typeof MODE_ALIASES)[number];
11
+
12
+ /** Public request shape before alias normalization. */
13
+ export type SiftLightRequest = Omit<SiftLightInput, "mode"> & {
14
+ mode?: SiftLightInput["mode"] | ModeAlias;
15
+ };
16
+
17
+ export interface NormalizedRequest {
18
+ input: SiftLightInput;
19
+ notes: string[];
20
+ }
21
+
22
+ /**
23
+ * The single place where unambiguous request shapes are rewritten before the
24
+ * mode contract runs. Every rewrite preserves the requested scope and evidence
25
+ * semantics and is reported back as a note; anything ambiguous still fails.
26
+ */
27
+ export function normalizeRequestAliases(request: SiftLightRequest): NormalizedRequest {
28
+ const notes: string[] = [];
29
+ const { mode, ...rest } = request;
30
+ let input: SiftLightInput = rest;
31
+ if (mode === "anyOf" || mode === "allOf") {
32
+ if (request[mode] === undefined)
33
+ throw new SiftLightError(`mode=${mode} is not a mode; pass ${mode}:[...terms] and omit mode`);
34
+ notes.push(`mode="${mode}" is not a mode; the ${mode} field alone selects this search.`);
35
+ } else if (mode === "summary" && request.anyOf !== undefined) {
36
+ notes.push(
37
+ "anyOf returns one analysis page whose header already carries per-file statistics; mode=summary was served by that page.",
38
+ );
39
+ } else if (mode !== undefined) {
40
+ input = { ...rest, mode };
41
+ }
42
+ if (input.sourceCursor !== undefined && input.line !== undefined) {
43
+ const { line: _line, ...withoutLine } = input;
44
+ input = withoutLine;
45
+ notes.push("line is ignored with sourceCursor; the continuation selects its own source range.");
46
+ }
47
+ return { input, notes };
48
+ }
@@ -62,7 +62,17 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
62
62
  auto: ordinaryFields,
63
63
  summary: ordinaryFields,
64
64
  matches: ordinaryFields,
65
- inspect: [...commonFields, "path", "line", "cursor", "matchIndex", "matchIndices", "targets"],
65
+ inspect: [
66
+ ...commonFields,
67
+ "path",
68
+ "paths",
69
+ "line",
70
+ "cursor",
71
+ "matchIndex",
72
+ "matchIndices",
73
+ "targets",
74
+ "scope",
75
+ ],
66
76
  outline: [...commonFields, "path", "line", "symbol", "cursor", "matchIndex", "maxFilesToParse"],
67
77
  imports: [
68
78
  ...commonFields,
@@ -82,7 +92,16 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
82
92
  "matchIndex",
83
93
  "maxFilesToParse",
84
94
  ],
85
- files: [...commonFields, "query", ...sourceFilters, "modifiedAfter", "modifiedBefore"],
95
+ files: [
96
+ ...commonFields,
97
+ "query",
98
+ ...sourceFilters,
99
+ "ignorePolicy",
100
+ "limit",
101
+ "scope",
102
+ "modifiedAfter",
103
+ "modifiedBefore",
104
+ ],
86
105
  structure: [...commonFields, "pattern", ...sourceFilters, "maxFilesToParse"],
87
106
  concept: [...commonFields, "query", ...sourceFilters, "maxFilesToParse"],
88
107
  hybrid: [...commonFields, "query", ...sourceFilters, "conceptLimit", "maxFilesToParse"],
@@ -92,9 +111,7 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
92
111
  cancel: ["mode", "operationId"],
93
112
  };
94
113
 
95
- export const SAFE_DROP_FIELDS: Partial<Record<SiftLightMode, readonly RequestField[]>> = {
96
- files: ["scope"],
97
- };
114
+ export const SAFE_DROP_FIELDS: Partial<Record<SiftLightMode, readonly RequestField[]>> = {};
98
115
 
99
116
  export const SUPPORTED_OUTLINE_EXTENSIONS = new Set(
100
117
  DEFAULT_LANGUAGE_CAPABILITIES.flatMap((descriptor) =>
@@ -149,9 +166,9 @@ export const REQUEST_FIELD_GUIDANCE: Partial<Record<RequestField, string>> = {
149
166
  context: "context is an output-context budget and is never silently dropped",
150
167
  limit: "limit is an output/page budget and is never silently dropped",
151
168
  scope:
152
- "scope applies to ordinary content search; mode=files rejects this field because its scope is fixed strict, and only redundant strict may be removed",
169
+ "scope applies to ordinary content search; mode=files is always strict, so it accepts only the redundant scope=strict",
153
170
  ignorePolicy:
154
- "respect keeps repository ignore rules; include searches ignored files but always excludes .git internals and protected paths",
171
+ "respect keeps repository ignore rules and discloses ignored files; include also searches or lists ignored files but always excludes .git internals and protected paths",
155
172
  patterns: "audit accepts named exact-literal patterns and returns one closure receipt",
156
173
  maxFilesToParse:
157
174
  "concept/hybrid automatically batch the requested scope; this optional field sets an advanced hard file ceiling",