sift-light 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +11 -4
- package/README.zh-CN.md +11 -4
- package/package.json +1 -1
- package/plugins/sift-light/.claude-plugin/plugin.json +1 -1
- package/plugins/sift-light/.codex-plugin/plugin.json +1 -1
- package/plugins/sift-light/.mcp.json +1 -1
- package/plugins/sift-light/concept-worker.mjs +39 -12
- package/plugins/sift-light/kimi.plugin.json +2 -2
- package/plugins/sift-light/omp-extension.mjs +558 -83
- package/plugins/sift-light/package.json +1 -1
- package/src/analysis-limits.ts +1 -1
- package/src/analysis-store.ts +6 -2
- package/src/analysis-types.ts +10 -1
- package/src/concept-inference.ts +44 -11
- package/src/concept-model.ts +2 -2
- package/src/concept-search.ts +12 -2
- package/src/concept-worker.mjs +39 -12
- package/src/concept-worker.ts +1 -1
- package/src/config-reader.ts +11 -3
- package/src/evidence-service.ts +117 -13
- package/src/file-discovery.ts +111 -17
- package/src/hybrid-search.ts +13 -0
- package/src/index.ts +4 -10
- package/src/inspect.ts +3 -0
- package/src/language-capability-definitions.ts +9 -0
- package/src/mcp-model-output.ts +47 -1
- package/src/mcp-semantic-judge.ts +19 -8
- package/src/mcp-server.mjs +754 -247
- package/src/mcp-server.ts +11 -13
- package/src/mcp.ts +2 -0
- package/src/model-error.ts +6 -16
- package/src/omp-index.ts +4 -6
- package/src/prompt-guidelines.ts +5 -5
- package/src/python-roles.ts +247 -0
- package/src/redaction.ts +9 -2
- package/src/request-aliases.ts +48 -0
- package/src/request-contract-catalog.ts +24 -7
- package/src/request-contract.ts +60 -6
- package/src/request.ts +3 -1
- package/src/runtime.ts +2 -2
- package/src/semantic-judge.ts +17 -3
- package/src/service.ts +31 -1
- package/src/session-summary.ts +4 -4
- package/src/source-inspection.ts +27 -2
- package/src/tool-schema.ts +18 -18
- package/src/tui/layout.ts +2 -2
- package/src/tui/renderers.ts +2 -2
- package/src/types.ts +4 -0
package/src/mcp-server.ts
CHANGED
|
@@ -8,10 +8,7 @@ import {
|
|
|
8
8
|
createDefaultSiftLightMcpService,
|
|
9
9
|
startSiftLightMcpServer,
|
|
10
10
|
} from "./mcp.js";
|
|
11
|
-
import {
|
|
12
|
-
createMcpSemanticJudgeIntegration,
|
|
13
|
-
mcpSemanticJudgeConfigSource,
|
|
14
|
-
} from "./mcp-semantic-judge.js";
|
|
11
|
+
import { createMcpSearchFeatures, mcpSemanticJudgeConfigSource } from "./mcp-semantic-judge.js";
|
|
15
12
|
import type { SemanticJudgeIntegration } from "./semantic-judge.js";
|
|
16
13
|
import { parseSiftLightMcpTransport, SIFT_LIGHT_MCP_USAGE } from "./mcp-cli.js";
|
|
17
14
|
import { parseSiftLightMcpOutputMode, type SiftLightMcpOutputMode } from "./mcp-output.js";
|
|
@@ -41,10 +38,6 @@ function allowedOrigins(): string[] {
|
|
|
41
38
|
.filter((origin) => origin.length > 0);
|
|
42
39
|
}
|
|
43
40
|
|
|
44
|
-
async function configuredSemanticJudge(): Promise<SemanticJudgeIntegration> {
|
|
45
|
-
return createMcpSemanticJudgeIntegration();
|
|
46
|
-
}
|
|
47
|
-
|
|
48
41
|
function logSemanticJudgeStatus(integration: SemanticJudgeIntegration): void {
|
|
49
42
|
const status = integration.config.enabled ? "enabled" : "disabled";
|
|
50
43
|
process.stderr.write(
|
|
@@ -55,12 +48,13 @@ function logSemanticJudgeStatus(integration: SemanticJudgeIntegration): void {
|
|
|
55
48
|
async function runHttpServer(
|
|
56
49
|
outputMode: SiftLightMcpOutputMode,
|
|
57
50
|
semanticJudge: SemanticJudgeIntegration | undefined,
|
|
51
|
+
vectorSearchEnabled: boolean,
|
|
58
52
|
): Promise<void> {
|
|
59
53
|
const running = await startSiftLightMcpServer({
|
|
60
54
|
cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.cwd(),
|
|
61
55
|
host: process.env.SIFT_LIGHT_MCP_HOST ?? DEFAULT_MCP_HOST,
|
|
62
56
|
port: environmentInteger("SIFT_LIGHT_MCP_PORT", DEFAULT_MCP_PORT, 0, 65_535),
|
|
63
|
-
createService: () => createDefaultSiftLightMcpService(semanticJudge),
|
|
57
|
+
createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled),
|
|
64
58
|
maxSessions: environmentInteger(
|
|
65
59
|
"SIFT_LIGHT_MCP_MAX_SESSIONS",
|
|
66
60
|
DEFAULT_MCP_MAX_SESSIONS,
|
|
@@ -108,11 +102,12 @@ async function runHttpServer(
|
|
|
108
102
|
async function runStdioServer(
|
|
109
103
|
outputMode: SiftLightMcpOutputMode,
|
|
110
104
|
semanticJudge: SemanticJudgeIntegration | undefined,
|
|
105
|
+
vectorSearchEnabled: boolean,
|
|
111
106
|
): Promise<void> {
|
|
112
107
|
const running = await startSiftLightMcpStdioServer({
|
|
113
108
|
cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.env.CLAUDE_PROJECT_DIR ?? process.cwd(),
|
|
114
109
|
outputMode,
|
|
115
|
-
createService: () => createDefaultSiftLightMcpService(semanticJudge),
|
|
110
|
+
createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled),
|
|
116
111
|
});
|
|
117
112
|
process.stderr.write("sift-light MCP serving one local client over stdio\n");
|
|
118
113
|
process.stderr.write(`sift-light MCP working directory: ${running.cwd}\n`);
|
|
@@ -140,13 +135,16 @@ async function main(): Promise<void> {
|
|
|
140
135
|
return;
|
|
141
136
|
}
|
|
142
137
|
const outputMode = parseSiftLightMcpOutputMode(process.env.SIFT_LIGHT_MCP_OUTPUT_MODE);
|
|
143
|
-
const semanticJudge = await
|
|
138
|
+
const { semanticJudge, vectorSearchEnabled } = await createMcpSearchFeatures();
|
|
144
139
|
logSemanticJudgeStatus(semanticJudge);
|
|
140
|
+
process.stderr.write(
|
|
141
|
+
`sift-light MCP vector search: ${vectorSearchEnabled ? "enabled" : "disabled"}\n`,
|
|
142
|
+
);
|
|
145
143
|
if (transport === "stdio") {
|
|
146
|
-
await runStdioServer(outputMode, semanticJudge);
|
|
144
|
+
await runStdioServer(outputMode, semanticJudge, vectorSearchEnabled);
|
|
147
145
|
return;
|
|
148
146
|
}
|
|
149
|
-
await runHttpServer(outputMode, semanticJudge);
|
|
147
|
+
await runHttpServer(outputMode, semanticJudge, vectorSearchEnabled);
|
|
150
148
|
}
|
|
151
149
|
|
|
152
150
|
try {
|
package/src/mcp.ts
CHANGED
|
@@ -93,12 +93,14 @@ export interface SiftLightMcpService {
|
|
|
93
93
|
|
|
94
94
|
export function createDefaultSiftLightMcpService(
|
|
95
95
|
semanticJudge?: SemanticJudgeIntegration,
|
|
96
|
+
vectorSearchEnabled = false,
|
|
96
97
|
): SiftLightMcpService {
|
|
97
98
|
const resolvedSemanticJudge =
|
|
98
99
|
semanticJudge ?? createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG);
|
|
99
100
|
return new SiftLightService({
|
|
100
101
|
runRipgrep: createRipgrepRunner(),
|
|
101
102
|
structure: createCtagsStructureProvider(),
|
|
103
|
+
vectorSearchEnabled,
|
|
102
104
|
semanticJudge: resolvedSemanticJudge,
|
|
103
105
|
});
|
|
104
106
|
}
|
package/src/model-error.ts
CHANGED
|
@@ -73,19 +73,14 @@ export interface RequestContractProjection {
|
|
|
73
73
|
text: string;
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
-
function projectRequestContract(
|
|
77
|
-
projected: RequestContractDetails,
|
|
78
|
-
serialized: string,
|
|
79
|
-
message: string,
|
|
80
|
-
): string {
|
|
81
|
-
const prefix = `sift-light failed: request rejected [${projected.code}]: ${message}`;
|
|
76
|
+
function projectRequestContract(projected: RequestContractDetails, serialized: string): string {
|
|
82
77
|
const recovery = projected.recovery;
|
|
83
78
|
const next = recovery.nextRequest
|
|
84
79
|
? "\nCopy the nested recovery.nextRequest object unchanged; do not repeat the original query."
|
|
85
|
-
: `\nRecovery action: ${recovery.action}
|
|
80
|
+
: `\nRecovery action: ${recovery.action}.`;
|
|
86
81
|
// Put the machine payload first so simple host adapters that locate the
|
|
87
82
|
// first `nextRequest` marker see the exact nested request object.
|
|
88
|
-
return `\nError details: ${serialized}
|
|
83
|
+
return `\nsift-light failed: request rejected [${projected.code}].\nError details: ${serialized}${next}`;
|
|
89
84
|
}
|
|
90
85
|
|
|
91
86
|
/** Build the one bounded contract projection consumed by structured and text hosts. */
|
|
@@ -94,11 +89,10 @@ export function requestContractProjection(
|
|
|
94
89
|
): RequestContractProjection {
|
|
95
90
|
const details = boundedRequestContractDetails(error.details);
|
|
96
91
|
const serializedDetails = JSON.stringify(details);
|
|
97
|
-
const
|
|
98
|
-
const result = projectRequestContract(details, serializedDetails, boundedMessage);
|
|
92
|
+
const result = projectRequestContract(details, serializedDetails);
|
|
99
93
|
if (Buffer.byteLength(result) <= MAX_REQUEST_RECOVERY_BYTES) return { details, text: result };
|
|
100
94
|
const compact: RequestContractDetails = {
|
|
101
|
-
code:
|
|
95
|
+
code: details.code,
|
|
102
96
|
...(details.mode ? { mode: details.mode } : {}),
|
|
103
97
|
issues: [
|
|
104
98
|
{
|
|
@@ -111,10 +105,6 @@ export function requestContractProjection(
|
|
|
111
105
|
reason: "Exact recovery was omitted; correct the request explicitly.",
|
|
112
106
|
},
|
|
113
107
|
};
|
|
114
|
-
const compactText = projectRequestContract(
|
|
115
|
-
compact,
|
|
116
|
-
JSON.stringify(compact),
|
|
117
|
-
"The request-contract error exceeded the bounded payload budget; correct the request explicitly.",
|
|
118
|
-
);
|
|
108
|
+
const compactText = projectRequestContract(compact, JSON.stringify(compact));
|
|
119
109
|
return { details: compact, text: compactText };
|
|
120
110
|
}
|
package/src/omp-index.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { homedir } from "node:os";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import {
|
|
4
|
-
DEFAULT_SEMANTIC_JUDGE_CONFIG,
|
|
5
4
|
normalizeSearchEnforcement,
|
|
6
5
|
readSiftLightConfigFile,
|
|
7
6
|
resolveSiftLightConfigPath,
|
|
@@ -27,7 +26,7 @@ import {
|
|
|
27
26
|
} from "./search-policy.js";
|
|
28
27
|
import { siftLightSchema } from "./tool-schema.js";
|
|
29
28
|
import { modelErrorText } from "./model-error.js";
|
|
30
|
-
import {
|
|
29
|
+
import { createConfiguredSemanticJudgeIntegration } from "./semantic-judge.js";
|
|
31
30
|
|
|
32
31
|
const SIFT_LIGHT_LABEL = "sift-light";
|
|
33
32
|
const OMP_REPLACED_SEARCH_TOOLS = new Set(["grep", "glob"]);
|
|
@@ -188,13 +187,12 @@ export async function registerOmpSiftLightExtension(
|
|
|
188
187
|
(await readSiftLightConfigFile(resolveSiftLightConfigPath(ompAgentDir()), {
|
|
189
188
|
missing: process.env[SIFT_LIGHT_CONFIG_ENV]?.trim() ? "error" : "defaults",
|
|
190
189
|
}));
|
|
191
|
-
const semanticJudge =
|
|
192
|
-
resolvedConfig.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG,
|
|
193
|
-
);
|
|
190
|
+
const semanticJudge = createConfiguredSemanticJudgeIntegration(resolvedConfig);
|
|
194
191
|
const runtime = new SiftLightRuntime(
|
|
195
192
|
new SiftLightService({
|
|
196
193
|
runRipgrep: createRipgrepRunner(),
|
|
197
194
|
structure: createCtagsStructureProvider(),
|
|
195
|
+
vectorSearchEnabled: resolvedConfig.vectorSearchEnabled === true,
|
|
198
196
|
semanticJudge,
|
|
199
197
|
}),
|
|
200
198
|
);
|
|
@@ -218,7 +216,7 @@ export async function registerOmpSiftLightExtension(
|
|
|
218
216
|
name: SIFT_LIGHT_LABEL,
|
|
219
217
|
label: SIFT_LIGHT_LABEL,
|
|
220
218
|
description:
|
|
221
|
-
"Search
|
|
219
|
+
"Search code with bounded evidence. Routine searches use pattern for fast exact content or mode=files with query for filenames; concept/hybrid require vectorSearchEnabled:true in sift-light.json.",
|
|
222
220
|
approval: "read",
|
|
223
221
|
promptSnippet: "Search file contents without flooding context",
|
|
224
222
|
promptGuidelines: siftLightPromptGuidelines(),
|
package/src/prompt-guidelines.ts
CHANGED
|
@@ -5,7 +5,7 @@ const SOURCE_OUTPUT_GUIDANCE =
|
|
|
5
5
|
|
|
6
6
|
export function siftLightPromptGuidelines(structuredOutput = true): string[] {
|
|
7
7
|
return [
|
|
8
|
-
`Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`,
|
|
8
|
+
`Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} For routine development searches, start with fast exact content, filename or applicable structural modes when the request has a usable name, symbol, error text or other literal clue. Omitted mode is ordinary exact search and never loads the local embedding model. Vector search is disabled by default; concept/hybrid require vectorSearchEnabled:true in sift-light.json and an installed model. They can take tens of seconds on an uncached scope, so select them only when semantic recall is needed. Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`,
|
|
9
9
|
`An omitted path searches the project cwd. Use scope:"strict" for a question restricted to one path; otherwise, if an explicit subpath has zero matches, ordinary and content-analysis searches retry from cwd and return project-wide counts with an expansion notice. Explicit absolute paths and .. traversal can search outside cwd, except protected external system areas and .git internals. Git changes mode remains cwd-scoped.`,
|
|
10
10
|
`Search output includes counts, categories, ranked paths, coverage and continuation metadata. Source excerpts may contain the searched text. Use mode="inspect" or the host read capability when exact source is required for an edit or verification.`,
|
|
11
11
|
`Use file and directory distributions to choose evidence. Reuse the visible cursor with path or paths for match metadata; mode="summary" pages the remaining file statistics. Match counts are not relevance scores.`,
|
|
@@ -14,7 +14,7 @@ export function siftLightPromptGuidelines(structuredOutput = true): string[] {
|
|
|
14
14
|
`Use anyOf:["term1","term2"] when every exact occurrence of 2-64 literals is needed in one version-bound result. It is case-sensitive, reports anonymized condition counts, and runs requests above eight terms as bounded parallel chunks. Large condition inventories have separate continuation pages; copy those requests to retrieve the complete counts.`,
|
|
15
15
|
`For a changed-code question, add changes:{base:"HEAD",scope:"lines",side:"new"}; omit target for the working tree, use side:"old" for deleted-side statistics. Copy returned continuation requests to preserve source versions.`,
|
|
16
16
|
`Use mode:"capabilities" when the language or requested operation is unclear to get a compact lazy inventory. Use mode:"outline" with a concrete source file path for symbol counts and locations, mode:"imports" for static relationships, and mode:"tests" for related-test candidates. Their text pages summarize metadata; structured details can retain source evidence.`,
|
|
17
|
-
`Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; business concepts
|
|
17
|
+
`Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; use hybrid/concept only for business concepts that cannot be located by a literal clue. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`,
|
|
18
18
|
`Use mode:"structure" only for JS/TS/TSX/Go, with a required nonempty ast-grep pattern such as "compare($X, $X)" or "send()" for code shapes across whitespace; the text page reports structural counts and locations; structured details can retain matched source evidence.`,
|
|
19
19
|
`Use mode:"concept" plus a natural-language query when names are unknown. It runs a pinned local multilingual model only after explicit installation; no search downloads weights or sends code to a remote model. Results expose candidate counts, score statistics, paths and the bounded ranked passage behind each candidate. Similarity scores identify candidates, not correctness.`,
|
|
20
20
|
`Use mode:"hybrid" plus query when wording may differ from the source. It reports exact and semantic counts separately, removes overlap, and retains one pageable evidence snapshot. conceptLimit changes only the semantic candidate count; retained candidates keep their bounded source passages.`,
|
|
@@ -28,9 +28,9 @@ export function siftLightPromptGuidelines(structuredOutput = true): string[] {
|
|
|
28
28
|
|
|
29
29
|
function siftLightModelGuidelines(): string[] {
|
|
30
30
|
return [
|
|
31
|
-
`Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE}
|
|
32
|
-
`Use ranked paths and
|
|
33
|
-
`Modes:
|
|
31
|
+
`Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Default search is exact and model-free. concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model; uncached runs may take tens of seconds. Omit mode/limit for auto pages.`,
|
|
32
|
+
`Use ranked paths and counts; Cursor continuation pages retained results. Use mode="inspect" for exact source before editing.`,
|
|
33
|
+
`Modes: files+query for filenames; anyOf/allOf for literals; outline/imports/tests for static code; structure for AST; concept/hybrid for semantic candidates. Similarity is not proof.`,
|
|
34
34
|
`On rejection keep the strongest applicable mode and apply the stated repair once. Do not repeat the rejected request or include its error. Only explicit capability-unavailable permits a visibly partial alternative.`,
|
|
35
35
|
];
|
|
36
36
|
}
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import type { SourceDocument } from "./source-document.js";
|
|
2
|
+
import type { SyntaxAnalysis, SyntaxRole } from "./syntax-types.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Bounded lexical role scanner for Python.
|
|
6
|
+
*
|
|
7
|
+
* It separates comments, string literals and code exactly, and marks `def`/`class`
|
|
8
|
+
* names, import statements and call-shaped callees. It does not parse Python:
|
|
9
|
+
* comments, strings and code are lexical facts ("syntax" certainty); a call is a
|
|
10
|
+
* name followed by `(` in code and therefore stays a "candidate". Offsets are
|
|
11
|
+
* UTF-16 indices into `document.text`, like every other role provider.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const KEYWORDS = new Set([
|
|
15
|
+
"and",
|
|
16
|
+
"as",
|
|
17
|
+
"assert",
|
|
18
|
+
"async",
|
|
19
|
+
"await",
|
|
20
|
+
"case",
|
|
21
|
+
"del",
|
|
22
|
+
"elif",
|
|
23
|
+
"else",
|
|
24
|
+
"except",
|
|
25
|
+
"for",
|
|
26
|
+
"from",
|
|
27
|
+
"global",
|
|
28
|
+
"if",
|
|
29
|
+
"import",
|
|
30
|
+
"in",
|
|
31
|
+
"is",
|
|
32
|
+
"lambda",
|
|
33
|
+
"match",
|
|
34
|
+
"nonlocal",
|
|
35
|
+
"not",
|
|
36
|
+
"or",
|
|
37
|
+
"raise",
|
|
38
|
+
"return",
|
|
39
|
+
"while",
|
|
40
|
+
"with",
|
|
41
|
+
"yield",
|
|
42
|
+
]);
|
|
43
|
+
const STRING_PREFIX = /^(?:[rRbBuUfF]|[rR][bBfF]|[bBfF][rR])?$/u;
|
|
44
|
+
const IDENTIFIER_START = /[\p{ID_Start}_]/u;
|
|
45
|
+
const IDENTIFIER_PART = /[\p{ID_Continue}]/u;
|
|
46
|
+
|
|
47
|
+
interface Span {
|
|
48
|
+
start: number;
|
|
49
|
+
end: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
interface Lexed {
|
|
53
|
+
comments: Span[];
|
|
54
|
+
strings: Span[];
|
|
55
|
+
/** `true` at each UTF-16 index that belongs to code (outside comments/strings). */
|
|
56
|
+
code: Uint8Array;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function lex(text: string): Lexed {
|
|
60
|
+
const comments: Span[] = [];
|
|
61
|
+
const strings: Span[] = [];
|
|
62
|
+
const code = new Uint8Array(text.length).fill(1);
|
|
63
|
+
const mark = (start: number, end: number): void => {
|
|
64
|
+
code.fill(0, start, end);
|
|
65
|
+
};
|
|
66
|
+
let index = 0;
|
|
67
|
+
while (index < text.length) {
|
|
68
|
+
const character = text[index];
|
|
69
|
+
if (character === "#") {
|
|
70
|
+
const newline = text.indexOf("\n", index);
|
|
71
|
+
const end = newline < 0 ? text.length : newline;
|
|
72
|
+
comments.push({ start: index, end });
|
|
73
|
+
mark(index, end);
|
|
74
|
+
index = end;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (character !== "'" && character !== '"') {
|
|
78
|
+
index += 1;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
// A string prefix (r, b, f, u and their pairs) is part of the literal.
|
|
82
|
+
let prefixStart = index;
|
|
83
|
+
while (prefixStart > 0 && /[rRbBuUfF]/u.test(text[prefixStart - 1] ?? "")) prefixStart -= 1;
|
|
84
|
+
const prefix = text.slice(prefixStart, index);
|
|
85
|
+
const beforePrefix = text[prefixStart - 1] ?? "";
|
|
86
|
+
const start =
|
|
87
|
+
STRING_PREFIX.test(prefix) && !IDENTIFIER_PART.test(beforePrefix) ? prefixStart : index;
|
|
88
|
+
const triple = text.slice(index, index + 3) === character.repeat(3);
|
|
89
|
+
const delimiter = triple ? character.repeat(3) : character;
|
|
90
|
+
let cursor = index + delimiter.length;
|
|
91
|
+
let end = text.length;
|
|
92
|
+
while (cursor < text.length) {
|
|
93
|
+
const current = text[cursor];
|
|
94
|
+
// A backslash always protects the next character from ending the literal,
|
|
95
|
+
// including in raw strings (where the backslash itself is kept).
|
|
96
|
+
if (current === "\\") {
|
|
97
|
+
cursor += 2;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (!triple && current === "\n") {
|
|
101
|
+
end = cursor;
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
if (text.startsWith(delimiter, cursor)) {
|
|
105
|
+
end = cursor + delimiter.length;
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
cursor += 1;
|
|
109
|
+
}
|
|
110
|
+
strings.push({ start, end });
|
|
111
|
+
mark(start, end);
|
|
112
|
+
index = end;
|
|
113
|
+
}
|
|
114
|
+
return { comments, strings, code };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function codeRanges(code: Uint8Array): Span[] {
|
|
118
|
+
const ranges: Span[] = [];
|
|
119
|
+
let start = -1;
|
|
120
|
+
for (let index = 0; index <= code.length; index += 1) {
|
|
121
|
+
const inCode = index < code.length && code[index] === 1;
|
|
122
|
+
if (inCode && start < 0) start = index;
|
|
123
|
+
else if (!inCode && start >= 0) {
|
|
124
|
+
ranges.push({ start, end: index });
|
|
125
|
+
start = -1;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return ranges;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function identifierEnd(text: string, start: number): number {
|
|
132
|
+
let end = start;
|
|
133
|
+
while (end < text.length && IDENTIFIER_PART.test(text[end] ?? "")) end += 1;
|
|
134
|
+
return end;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function skipSpaces(text: string, index: number, code: Uint8Array): number {
|
|
138
|
+
let cursor = index;
|
|
139
|
+
while (cursor < text.length && code[cursor] === 1 && /[ \t]/u.test(text[cursor] ?? ""))
|
|
140
|
+
cursor += 1;
|
|
141
|
+
return cursor;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Index after the statement starting at `start`, following parenthesized continuation lines. */
|
|
145
|
+
function statementEnd(text: string, start: number, code: Uint8Array): number {
|
|
146
|
+
let depth = 0;
|
|
147
|
+
for (let index = start; index < text.length; index += 1) {
|
|
148
|
+
if (code[index] !== 1) continue;
|
|
149
|
+
const character = text[index];
|
|
150
|
+
if (character === "(" || character === "[" || character === "{") depth += 1;
|
|
151
|
+
else if (character === ")" || character === "]" || character === "}")
|
|
152
|
+
depth = Math.max(0, depth - 1);
|
|
153
|
+
else if (character === "\\" && text[index + 1] === "\n") index += 1;
|
|
154
|
+
else if (character === "\n" && depth === 0) return index;
|
|
155
|
+
}
|
|
156
|
+
return text.length;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function pythonRoleAnalysis(document: SourceDocument): SyntaxAnalysis {
|
|
160
|
+
const text = document.text;
|
|
161
|
+
const { comments, strings, code } = lex(text);
|
|
162
|
+
const roles: SyntaxRole[] = [];
|
|
163
|
+
const push = (
|
|
164
|
+
start: number,
|
|
165
|
+
end: number,
|
|
166
|
+
role: SyntaxRole["role"],
|
|
167
|
+
certainty: SyntaxRole["certainty"],
|
|
168
|
+
subkind?: string,
|
|
169
|
+
): void => {
|
|
170
|
+
if (start < end)
|
|
171
|
+
roles.push({ start, end, role, certainty, node: 0, ...(subkind ? { subkind } : {}) });
|
|
172
|
+
};
|
|
173
|
+
for (const span of comments) push(span.start, span.end, "comment", "syntax");
|
|
174
|
+
for (const span of strings) push(span.start, span.end, "string", "syntax");
|
|
175
|
+
for (const span of codeRanges(code)) push(span.start, span.end, "code", "syntax");
|
|
176
|
+
|
|
177
|
+
let index = 0;
|
|
178
|
+
let lineStart = true;
|
|
179
|
+
while (index < text.length) {
|
|
180
|
+
const character = text[index] ?? "";
|
|
181
|
+
if (character === "\n") {
|
|
182
|
+
lineStart = true;
|
|
183
|
+
index += 1;
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (code[index] !== 1 || !IDENTIFIER_START.test(character)) {
|
|
187
|
+
if (!/[ \t]/u.test(character)) lineStart = false;
|
|
188
|
+
index += 1;
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const previous = text[index - 1] ?? "";
|
|
192
|
+
if (IDENTIFIER_PART.test(previous)) {
|
|
193
|
+
index += 1;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const end = identifierEnd(text, index);
|
|
197
|
+
const word = text.slice(index, end);
|
|
198
|
+
const atStatementStart = lineStart;
|
|
199
|
+
lineStart = false;
|
|
200
|
+
if (atStatementStart && (word === "import" || word === "from")) {
|
|
201
|
+
const statement = statementEnd(text, index, code);
|
|
202
|
+
const body = text.slice(index, statement);
|
|
203
|
+
if (word === "import" || /\bimport\b/u.test(body)) {
|
|
204
|
+
push(index, statement, "import", "syntax", word === "from" ? "from-import" : "import");
|
|
205
|
+
index = statement;
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (word === "def" || word === "class") {
|
|
210
|
+
const nameStart = skipSpaces(text, end, code);
|
|
211
|
+
if (IDENTIFIER_START.test(text[nameStart] ?? "")) {
|
|
212
|
+
const nameEnd = identifierEnd(text, nameStart);
|
|
213
|
+
push(nameStart, nameEnd, "declaration", "syntax", word === "def" ? "function" : "class");
|
|
214
|
+
index = nameEnd;
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
// Attribute chains such as `client.fetch(` are one callee.
|
|
219
|
+
let chainEnd = end;
|
|
220
|
+
while (text[chainEnd] === "." && IDENTIFIER_START.test(text[chainEnd + 1] ?? "")) {
|
|
221
|
+
chainEnd = identifierEnd(text, chainEnd + 1);
|
|
222
|
+
}
|
|
223
|
+
const open = skipSpaces(text, chainEnd, code);
|
|
224
|
+
const lastSegment = text.slice(text.lastIndexOf(".", chainEnd - 1) + 1, chainEnd);
|
|
225
|
+
const before = text.slice(Math.max(0, index - 4), index);
|
|
226
|
+
if (
|
|
227
|
+
text[open] === "(" &&
|
|
228
|
+
code[open] === 1 &&
|
|
229
|
+
!KEYWORDS.has(word) &&
|
|
230
|
+
!KEYWORDS.has(lastSegment) &&
|
|
231
|
+
!/\bdef\s+$|\bclass\s+$/u.test(before)
|
|
232
|
+
) {
|
|
233
|
+
push(index, open + 1, "call", "candidate", "lexical-call");
|
|
234
|
+
}
|
|
235
|
+
index = chainEnd;
|
|
236
|
+
}
|
|
237
|
+
roles.sort((a, b) => a.start - b.start || a.end - b.end || a.role.localeCompare(b.role));
|
|
238
|
+
return {
|
|
239
|
+
status: "ok",
|
|
240
|
+
nodes: [],
|
|
241
|
+
children: [],
|
|
242
|
+
symbols: [],
|
|
243
|
+
roles,
|
|
244
|
+
diagnostics: [],
|
|
245
|
+
limited: false,
|
|
246
|
+
};
|
|
247
|
+
}
|
package/src/redaction.ts
CHANGED
|
@@ -7,6 +7,10 @@ const SENSITIVE_ASSIGNMENT = new RegExp(
|
|
|
7
7
|
"gi",
|
|
8
8
|
);
|
|
9
9
|
const SENSITIVE_TOKEN = /\b(?:sk|ghp|xox[baprs])[-_][A-Za-z0-9][A-Za-z0-9_-]{7,}\b/g;
|
|
10
|
+
// An unquoted value that starts as a call or index expression (`env.get(`,
|
|
11
|
+
// `os.environ[`) is source code that reads a secret, not the secret itself.
|
|
12
|
+
// Masking it hides the evidence a reader needs, such as which key is read.
|
|
13
|
+
const CODE_EXPRESSION = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*[([]/u;
|
|
10
14
|
const TYPE_ONLY_VALUES = new Set([
|
|
11
15
|
"boolean",
|
|
12
16
|
"number",
|
|
@@ -27,9 +31,12 @@ function redactString(value: string): { value: string; count: number } {
|
|
|
27
31
|
SENSITIVE_ASSIGNMENT,
|
|
28
32
|
(match: string, prefix: string, rawValue: string) => {
|
|
29
33
|
const unquoted = rawValue.replace(/^["']|["']$/g, "").toLowerCase();
|
|
30
|
-
if (TYPE_ONLY_VALUES.has(unquoted)) return match;
|
|
34
|
+
if (TYPE_ONLY_VALUES.has(unquoted) || CODE_EXPRESSION.test(rawValue)) return match;
|
|
31
35
|
count += 1;
|
|
32
|
-
|
|
36
|
+
// Keep the source's own quoting: adding quotes to an unquoted value would
|
|
37
|
+
// misrepresent the file (for example, a bare `.env` value shown as quoted).
|
|
38
|
+
const quote = rawValue.startsWith('"') || rawValue.startsWith("'") ? rawValue[0] : "";
|
|
39
|
+
return `${prefix}${quote}[REDACTED]${quote}`;
|
|
33
40
|
},
|
|
34
41
|
);
|
|
35
42
|
redacted = redacted.replace(SENSITIVE_TOKEN, () => {
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { SiftLightError } from "./errors.js";
|
|
2
|
+
import type { SiftLightInput } from "./service.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Field names that callers commonly send as a mode. They are not modes: the field
|
|
6
|
+
* alone selects the multi-term search. The schema accepts them so the request can
|
|
7
|
+
* be served, and the result discloses the normalization.
|
|
8
|
+
*/
|
|
9
|
+
export const MODE_ALIASES = ["anyOf", "allOf"] as const;
|
|
10
|
+
export type ModeAlias = (typeof MODE_ALIASES)[number];
|
|
11
|
+
|
|
12
|
+
/** Public request shape before alias normalization. */
|
|
13
|
+
export type SiftLightRequest = Omit<SiftLightInput, "mode"> & {
|
|
14
|
+
mode?: SiftLightInput["mode"] | ModeAlias;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export interface NormalizedRequest {
|
|
18
|
+
input: SiftLightInput;
|
|
19
|
+
notes: string[];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The single place where unambiguous request shapes are rewritten before the
|
|
24
|
+
* mode contract runs. Every rewrite preserves the requested scope and evidence
|
|
25
|
+
* semantics and is reported back as a note; anything ambiguous still fails.
|
|
26
|
+
*/
|
|
27
|
+
export function normalizeRequestAliases(request: SiftLightRequest): NormalizedRequest {
|
|
28
|
+
const notes: string[] = [];
|
|
29
|
+
const { mode, ...rest } = request;
|
|
30
|
+
let input: SiftLightInput = rest;
|
|
31
|
+
if (mode === "anyOf" || mode === "allOf") {
|
|
32
|
+
if (request[mode] === undefined)
|
|
33
|
+
throw new SiftLightError(`mode=${mode} is not a mode; pass ${mode}:[...terms] and omit mode`);
|
|
34
|
+
notes.push(`mode="${mode}" is not a mode; the ${mode} field alone selects this search.`);
|
|
35
|
+
} else if (mode === "summary" && request.anyOf !== undefined) {
|
|
36
|
+
notes.push(
|
|
37
|
+
"anyOf returns one analysis page whose header already carries per-file statistics; mode=summary was served by that page.",
|
|
38
|
+
);
|
|
39
|
+
} else if (mode !== undefined) {
|
|
40
|
+
input = { ...rest, mode };
|
|
41
|
+
}
|
|
42
|
+
if (input.sourceCursor !== undefined && input.line !== undefined) {
|
|
43
|
+
const { line: _line, ...withoutLine } = input;
|
|
44
|
+
input = withoutLine;
|
|
45
|
+
notes.push("line is ignored with sourceCursor; the continuation selects its own source range.");
|
|
46
|
+
}
|
|
47
|
+
return { input, notes };
|
|
48
|
+
}
|
|
@@ -62,7 +62,17 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
|
|
|
62
62
|
auto: ordinaryFields,
|
|
63
63
|
summary: ordinaryFields,
|
|
64
64
|
matches: ordinaryFields,
|
|
65
|
-
inspect: [
|
|
65
|
+
inspect: [
|
|
66
|
+
...commonFields,
|
|
67
|
+
"path",
|
|
68
|
+
"paths",
|
|
69
|
+
"line",
|
|
70
|
+
"cursor",
|
|
71
|
+
"matchIndex",
|
|
72
|
+
"matchIndices",
|
|
73
|
+
"targets",
|
|
74
|
+
"scope",
|
|
75
|
+
],
|
|
66
76
|
outline: [...commonFields, "path", "line", "symbol", "cursor", "matchIndex", "maxFilesToParse"],
|
|
67
77
|
imports: [
|
|
68
78
|
...commonFields,
|
|
@@ -82,7 +92,16 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
|
|
|
82
92
|
"matchIndex",
|
|
83
93
|
"maxFilesToParse",
|
|
84
94
|
],
|
|
85
|
-
files: [
|
|
95
|
+
files: [
|
|
96
|
+
...commonFields,
|
|
97
|
+
"query",
|
|
98
|
+
...sourceFilters,
|
|
99
|
+
"ignorePolicy",
|
|
100
|
+
"limit",
|
|
101
|
+
"scope",
|
|
102
|
+
"modifiedAfter",
|
|
103
|
+
"modifiedBefore",
|
|
104
|
+
],
|
|
86
105
|
structure: [...commonFields, "pattern", ...sourceFilters, "maxFilesToParse"],
|
|
87
106
|
concept: [...commonFields, "query", ...sourceFilters, "maxFilesToParse"],
|
|
88
107
|
hybrid: [...commonFields, "query", ...sourceFilters, "conceptLimit", "maxFilesToParse"],
|
|
@@ -92,9 +111,7 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
|
|
|
92
111
|
cancel: ["mode", "operationId"],
|
|
93
112
|
};
|
|
94
113
|
|
|
95
|
-
export const SAFE_DROP_FIELDS: Partial<Record<SiftLightMode, readonly RequestField[]>> = {
|
|
96
|
-
files: ["scope"],
|
|
97
|
-
};
|
|
114
|
+
export const SAFE_DROP_FIELDS: Partial<Record<SiftLightMode, readonly RequestField[]>> = {};
|
|
98
115
|
|
|
99
116
|
export const SUPPORTED_OUTLINE_EXTENSIONS = new Set(
|
|
100
117
|
DEFAULT_LANGUAGE_CAPABILITIES.flatMap((descriptor) =>
|
|
@@ -149,9 +166,9 @@ export const REQUEST_FIELD_GUIDANCE: Partial<Record<RequestField, string>> = {
|
|
|
149
166
|
context: "context is an output-context budget and is never silently dropped",
|
|
150
167
|
limit: "limit is an output/page budget and is never silently dropped",
|
|
151
168
|
scope:
|
|
152
|
-
"scope applies to ordinary content search; mode=files
|
|
169
|
+
"scope applies to ordinary content search; mode=files is always strict, so it accepts only the redundant scope=strict",
|
|
153
170
|
ignorePolicy:
|
|
154
|
-
"respect keeps repository ignore rules; include searches ignored files but always excludes .git internals and protected paths",
|
|
171
|
+
"respect keeps repository ignore rules and discloses ignored files; include also searches or lists ignored files but always excludes .git internals and protected paths",
|
|
155
172
|
patterns: "audit accepts named exact-literal patterns and returns one closure receipt",
|
|
156
173
|
maxFilesToParse:
|
|
157
174
|
"concept/hybrid automatically batch the requested scope; this optional field sets an advanced hard file ceiling",
|