sift-light 1.0.1 → 1.0.2

package/src/python-roles.ts

@@ -0,0 +1,247 @@
+import type { SourceDocument } from "./source-document.js";
+import type { SyntaxAnalysis, SyntaxRole } from "./syntax-types.js";
+
+/**
+ * Bounded lexical role scanner for Python.
+ *
+ * It separates comments, string literals and code exactly, and marks `def`/`class`
+ * names, import statements and call-shaped callees. It does not parse Python:
+ * comments, strings and code are lexical facts ("syntax" certainty); a call is a
+ * name followed by `(` in code and therefore stays a "candidate". Offsets are
+ * UTF-16 indices into `document.text`, like every other role provider.
+ */
+
+const KEYWORDS = new Set([
+  "and",
+  "as",
+  "assert",
+  "async",
+  "await",
+  "case",
+  "del",
+  "elif",
+  "else",
+  "except",
+  "for",
+  "from",
+  "global",
+  "if",
+  "import",
+  "in",
+  "is",
+  "lambda",
+  "match",
+  "nonlocal",
+  "not",
+  "or",
+  "raise",
+  "return",
+  "while",
+  "with",
+  "yield",
+]);
+const STRING_PREFIX = /^(?:[rRbBuUfF]|[rR][bBfF]|[bBfF][rR])?$/u;
+const IDENTIFIER_START = /[\p{ID_Start}_]/u;
+const IDENTIFIER_PART = /[\p{ID_Continue}]/u;
+
+interface Span {
+  start: number;
+  end: number;
+}
+
+interface Lexed {
+  comments: Span[];
+  strings: Span[];
+  /** `true` at each UTF-16 index that belongs to code (outside comments/strings). */
+  code: Uint8Array;
+}
+
+function lex(text: string): Lexed {
+  const comments: Span[] = [];
+  const strings: Span[] = [];
+  const code = new Uint8Array(text.length).fill(1);
+  const mark = (start: number, end: number): void => {
+    code.fill(0, start, end);
+  };
+  let index = 0;
+  while (index < text.length) {
+    const character = text[index];
+    if (character === "#") {
+      const newline = text.indexOf("\n", index);
+      const end = newline < 0 ? text.length : newline;
+      comments.push({ start: index, end });
+      mark(index, end);
+      index = end;
+      continue;
+    }
+    if (character !== "'" && character !== '"') {
+      index += 1;
+      continue;
+    }
+    // A string prefix (r, b, f, u and their pairs) is part of the literal.
+    let prefixStart = index;
+    while (prefixStart > 0 && /[rRbBuUfF]/u.test(text[prefixStart - 1] ?? "")) prefixStart -= 1;
+    const prefix = text.slice(prefixStart, index);
+    const beforePrefix = text[prefixStart - 1] ?? "";
+    const start =
+      STRING_PREFIX.test(prefix) && !IDENTIFIER_PART.test(beforePrefix) ? prefixStart : index;
+    const triple = text.slice(index, index + 3) === character.repeat(3);
+    const delimiter = triple ? character.repeat(3) : character;
+    let cursor = index + delimiter.length;
+    let end = text.length;
+    while (cursor < text.length) {
+      const current = text[cursor];
+      // A backslash always protects the next character from ending the literal,
+      // including in raw strings (where the backslash itself is kept).
+      if (current === "\\") {
+        cursor += 2;
+        continue;
+      }
+      if (!triple && current === "\n") {
+        end = cursor;
+        break;
+      }
+      if (text.startsWith(delimiter, cursor)) {
+        end = cursor + delimiter.length;
+        break;
+      }
+      cursor += 1;
+    }
+    strings.push({ start, end });
+    mark(start, end);
+    index = end;
+  }
+  return { comments, strings, code };
+}
+
+function codeRanges(code: Uint8Array): Span[] {
+  const ranges: Span[] = [];
+  let start = -1;
+  for (let index = 0; index <= code.length; index += 1) {
+    const inCode = index < code.length && code[index] === 1;
+    if (inCode && start < 0) start = index;
+    else if (!inCode && start >= 0) {
+      ranges.push({ start, end: index });
+      start = -1;
+    }
+  }
+  return ranges;
+}
+
+function identifierEnd(text: string, start: number): number {
+  let end = start;
+  while (end < text.length && IDENTIFIER_PART.test(text[end] ?? "")) end += 1;
+  return end;
+}
+
+function skipSpaces(text: string, index: number, code: Uint8Array): number {
+  let cursor = index;
+  while (cursor < text.length && code[cursor] === 1 && /[ \t]/u.test(text[cursor] ?? ""))
+    cursor += 1;
+  return cursor;
+}
+
+/** Index after the statement starting at `start`, following parenthesized continuation lines. */
+function statementEnd(text: string, start: number, code: Uint8Array): number {
+  let depth = 0;
+  for (let index = start; index < text.length; index += 1) {
+    if (code[index] !== 1) continue;
+    const character = text[index];
+    if (character === "(" || character === "[" || character === "{") depth += 1;
+    else if (character === ")" || character === "]" || character === "}")
+      depth = Math.max(0, depth - 1);
+    else if (character === "\\" && text[index + 1] === "\n") index += 1;
+    else if (character === "\n" && depth === 0) return index;
+  }
+  return text.length;
+}
+
+export function pythonRoleAnalysis(document: SourceDocument): SyntaxAnalysis {
+  const text = document.text;
+  const { comments, strings, code } = lex(text);
+  const roles: SyntaxRole[] = [];
+  const push = (
+    start: number,
+    end: number,
+    role: SyntaxRole["role"],
+    certainty: SyntaxRole["certainty"],
+    subkind?: string,
+  ): void => {
+    if (start < end)
+      roles.push({ start, end, role, certainty, node: 0, ...(subkind ? { subkind } : {}) });
+  };
+  for (const span of comments) push(span.start, span.end, "comment", "syntax");
+  for (const span of strings) push(span.start, span.end, "string", "syntax");
+  for (const span of codeRanges(code)) push(span.start, span.end, "code", "syntax");
+
+  let index = 0;
+  let lineStart = true;
+  while (index < text.length) {
+    const character = text[index] ?? "";
+    if (character === "\n") {
+      lineStart = true;
+      index += 1;
+      continue;
+    }
+    if (code[index] !== 1 || !IDENTIFIER_START.test(character)) {
+      if (!/[ \t]/u.test(character)) lineStart = false;
+      index += 1;
+      continue;
+    }
+    const previous = text[index - 1] ?? "";
+    if (IDENTIFIER_PART.test(previous)) {
+      index += 1;
+      continue;
+    }
+    const end = identifierEnd(text, index);
+    const word = text.slice(index, end);
+    const atStatementStart = lineStart;
+    lineStart = false;
+    if (atStatementStart && (word === "import" || word === "from")) {
+      const statement = statementEnd(text, index, code);
+      const body = text.slice(index, statement);
+      if (word === "import" || /\bimport\b/u.test(body)) {
+        push(index, statement, "import", "syntax", word === "from" ? "from-import" : "import");
+        index = statement;
+        continue;
+      }
+    }
+    if (word === "def" || word === "class") {
+      const nameStart = skipSpaces(text, end, code);
+      if (IDENTIFIER_START.test(text[nameStart] ?? "")) {
+        const nameEnd = identifierEnd(text, nameStart);
+        push(nameStart, nameEnd, "declaration", "syntax", word === "def" ? "function" : "class");
+        index = nameEnd;
+        continue;
+      }
+    }
+    // Attribute chains such as `client.fetch(` are one callee.
+    let chainEnd = end;
+    while (text[chainEnd] === "." && IDENTIFIER_START.test(text[chainEnd + 1] ?? "")) {
+      chainEnd = identifierEnd(text, chainEnd + 1);
+    }
+    const open = skipSpaces(text, chainEnd, code);
+    const lastSegment = text.slice(text.lastIndexOf(".", chainEnd - 1) + 1, chainEnd);
+    const before = text.slice(Math.max(0, index - 4), index);
+    if (
+      text[open] === "(" &&
+      code[open] === 1 &&
+      !KEYWORDS.has(word) &&
+      !KEYWORDS.has(lastSegment) &&
+      !/\bdef\s+$|\bclass\s+$/u.test(before)
+    ) {
+      push(index, open + 1, "call", "candidate", "lexical-call");
+    }
+    index = chainEnd;
+  }
+  roles.sort((a, b) => a.start - b.start || a.end - b.end || a.role.localeCompare(b.role));
+  return {
+    status: "ok",
+    nodes: [],
+    children: [],
+    symbols: [],
+    roles,
+    diagnostics: [],
+    limited: false,
+  };
+}

package/src/redaction.ts

@@ -7,6 +7,10 @@ const SENSITIVE_ASSIGNMENT = new RegExp(
   "gi",
 );
 const SENSITIVE_TOKEN = /\b(?:sk|ghp|xox[baprs])[-_][A-Za-z0-9][A-Za-z0-9_-]{7,}\b/g;
+// An unquoted value that starts as a call or index expression (`env.get(`,
+// `os.environ[`) is source code that reads a secret, not the secret itself.
+// Masking it hides the evidence a reader needs, such as which key is read.
+const CODE_EXPRESSION = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*[([]/u;
 const TYPE_ONLY_VALUES = new Set([
   "boolean",
   "number",
@@ -27,9 +31,12 @@ function redactString(value: string): { value: string; count: number } {
     SENSITIVE_ASSIGNMENT,
     (match: string, prefix: string, rawValue: string) => {
       const unquoted = rawValue.replace(/^["']|["']$/g, "").toLowerCase();
-      if (TYPE_ONLY_VALUES.has(unquoted)) return match;
+      if (TYPE_ONLY_VALUES.has(unquoted) || CODE_EXPRESSION.test(rawValue)) return match;
       count += 1;
-      return `${prefix}"[REDACTED]"`;
+      // Keep the source's own quoting: adding quotes to an unquoted value would
+      // misrepresent the file (for example, a bare `.env` value shown as quoted).
+      const quote = rawValue.startsWith('"') || rawValue.startsWith("'") ? rawValue[0] : "";
+      return `${prefix}${quote}[REDACTED]${quote}`;
     },
   );
   redacted = redacted.replace(SENSITIVE_TOKEN, () => {

package/src/request-aliases.ts

@@ -0,0 +1,48 @@
+import { SiftLightError } from "./errors.js";
+import type { SiftLightInput } from "./service.js";
+
+/**
+ * Field names that callers commonly send as a mode. They are not modes: the field
+ * alone selects the multi-term search. The schema accepts them so the request can
+ * be served, and the result discloses the normalization.
+ */
+export const MODE_ALIASES = ["anyOf", "allOf"] as const;
+export type ModeAlias = (typeof MODE_ALIASES)[number];
+
+/** Public request shape before alias normalization. */
+export type SiftLightRequest = Omit<SiftLightInput, "mode"> & {
+  mode?: SiftLightInput["mode"] | ModeAlias;
+};
+
+export interface NormalizedRequest {
+  input: SiftLightInput;
+  notes: string[];
+}
+
+/**
+ * The single place where unambiguous request shapes are rewritten before the
+ * mode contract runs. Every rewrite preserves the requested scope and evidence
+ * semantics and is reported back as a note; anything ambiguous still fails.
+ */
+export function normalizeRequestAliases(request: SiftLightRequest): NormalizedRequest {
+  const notes: string[] = [];
+  const { mode, ...rest } = request;
+  let input: SiftLightInput = rest;
+  if (mode === "anyOf" || mode === "allOf") {
+    if (request[mode] === undefined)
+      throw new SiftLightError(`mode=${mode} is not a mode; pass ${mode}:[...terms] and omit mode`);
+    notes.push(`mode="${mode}" is not a mode; the ${mode} field alone selects this search.`);
+  } else if (mode === "summary" && request.anyOf !== undefined) {
+    notes.push(
+      "anyOf returns one analysis page whose header already carries per-file statistics; mode=summary was served by that page.",
+    );
+  } else if (mode !== undefined) {
+    input = { ...rest, mode };
+  }
+  if (input.sourceCursor !== undefined && input.line !== undefined) {
+    const { line: _line, ...withoutLine } = input;
+    input = withoutLine;
+    notes.push("line is ignored with sourceCursor; the continuation selects its own source range.");
+  }
+  return { input, notes };
+}

package/src/request-contract-catalog.ts

@@ -62,7 +62,17 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
   auto: ordinaryFields,
   summary: ordinaryFields,
   matches: ordinaryFields,
-  inspect: [...commonFields, "path", "line", "cursor", "matchIndex", "matchIndices", "targets"],
+  inspect: [
+    ...commonFields,
+    "path",
+    "paths",
+    "line",
+    "cursor",
+    "matchIndex",
+    "matchIndices",
+    "targets",
+    "scope",
+  ],
   outline: [...commonFields, "path", "line", "symbol", "cursor", "matchIndex", "maxFilesToParse"],
   imports: [
     ...commonFields,
@@ -82,7 +92,16 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
     "matchIndex",
     "maxFilesToParse",
   ],
-  files: [...commonFields, "query", ...sourceFilters, "modifiedAfter", "modifiedBefore"],
+  files: [
+    ...commonFields,
+    "query",
+    ...sourceFilters,
+    "ignorePolicy",
+    "limit",
+    "scope",
+    "modifiedAfter",
+    "modifiedBefore",
+  ],
   structure: [...commonFields, "pattern", ...sourceFilters, "maxFilesToParse"],
   concept: [...commonFields, "query", ...sourceFilters, "maxFilesToParse"],
   hybrid: [...commonFields, "query", ...sourceFilters, "conceptLimit", "maxFilesToParse"],
@@ -92,9 +111,7 @@ export const MODE_FIELDS_BY_MODE: Record<SiftLightMode, readonly RequestField[]>
   cancel: ["mode", "operationId"],
 };
 
-export const SAFE_DROP_FIELDS: Partial<Record<SiftLightMode, readonly RequestField[]>> = {
-  files: ["scope"],
-};
+export const SAFE_DROP_FIELDS: Partial<Record<SiftLightMode, readonly RequestField[]>> = {};
 
 export const SUPPORTED_OUTLINE_EXTENSIONS = new Set(
   DEFAULT_LANGUAGE_CAPABILITIES.flatMap((descriptor) =>
@@ -149,9 +166,9 @@ export const REQUEST_FIELD_GUIDANCE: Partial<Record<RequestField, string>> = {
   context: "context is an output-context budget and is never silently dropped",
   limit: "limit is an output/page budget and is never silently dropped",
   scope:
-    "scope applies to ordinary content search; mode=files rejects this field because its scope is fixed strict, and only redundant strict may be removed",
+    "scope applies to ordinary content search; mode=files is always strict, so it accepts only the redundant scope=strict",
   ignorePolicy:
-    "respect keeps repository ignore rules; include searches ignored files but always excludes .git internals and protected paths",
+    "respect keeps repository ignore rules and discloses ignored files; include also searches or lists ignored files but always excludes .git internals and protected paths",
   patterns: "audit accepts named exact-literal patterns and returns one closure receipt",
   maxFilesToParse:
     "concept/hybrid automatically batch the requested scope; this optional field sets an advanced hard file ceiling",

package/src/request-contract.ts

@@ -132,6 +132,21 @@ function plainFilePatternRecovery(
   );
 }
 
+/** `pattern: ".*"` in files mode means "every file": the exact repair omits it. */
+function matchAllFilePatternRecovery(
+  mode: SiftLightMode,
+  input: Record<string, unknown>,
+  invalid: readonly string[],
+): boolean {
+  return (
+    mode === "files" &&
+    invalid.includes("pattern") &&
+    input.query === undefined &&
+    typeof input.pattern === "string" &&
+    /^(?:\.\*|\.\+|\*+)$/u.test(input.pattern.trim())
+  );
+}
+
 function safeNextRequest(
   input: Record<string, unknown>,
   mode: SiftLightMode,
@@ -140,7 +155,7 @@ function safeNextRequest(
   if (input.redact === true && containsSensitiveText(input)) return undefined;
   const safe = new Set<string>(SAFE_DROP_FIELDS[mode] ?? []);
   const plainFilePattern = plainFilePatternRecovery(mode, input, invalid);
-  if (plainFilePattern) safe.add("pattern");
+  if (plainFilePattern || matchAllFilePatternRecovery(mode, input, invalid)) safe.add("pattern");
   if (invalid.some((field) => !safe.has(field))) return undefined;
   const next: Record<string, unknown> = { ...input };
   for (const field of invalid) delete next[field];
@@ -203,7 +218,9 @@ function fieldsError(
             action: "retry",
             reason: plainFilePatternRecovery(mode, input, invalid)
               ? "For filename discovery, move the plain text from pattern to query and copy nextRequest; all filters are preserved."
-              : `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`,
+              : matchAllFilePatternRecovery(mode, input, invalid)
+                ? "A match-all pattern lists every file; omitting query does that, so copy nextRequest; all filters are preserved."
+                : `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`,
             nextRequest,
           }
         : {
@@ -458,7 +475,8 @@ export function validateRequestContract(input: SiftLightInput): void {
   const allowed = new Set(MODE_FIELDS_BY_MODE[mode]);
   if (mode === "inspect" && raw.sourceCursor !== undefined) {
     allowed.clear();
-    for (const field of ["mode", "sourceCursor", "redact"] as const) allowed.add(field);
+    // path is checked against the continuation's own source; it cannot redirect it.
+    for (const field of ["mode", "sourceCursor", "redact", "path"] as const) allowed.add(field);
   }
   if (
     (mode === "auto" || mode === "summary" || mode === "matches") &&

package/src/request.ts

@@ -108,7 +108,9 @@ export function normalizeRequest(input: RawSearchInput): SearchRequest {
     throw new SiftLightError("ignorePolicy must be respect or include");
   const pattern = input.pattern;
   if (pattern === undefined) {
-    throw new SiftLightError("pattern is required when cursor is not provided");
+    throw new SiftLightError(
+      'pattern is required when cursor is not provided; to list files use mode="files" (query optional), to read a known file use mode="inspect" with path',
+    );
   }
 
   const path = input.path?.replace(/^@/, "");

package/src/runtime.ts

@@ -1,5 +1,5 @@
 import type { SiftLightLocale } from "./config.js";
-import type { SiftLightInput, SiftLightSearchOptions, SiftLightService } from "./service.js";
+import type { SiftLightRequest, SiftLightSearchOptions, SiftLightService } from "./service.js";
 import { type SessionSummarySnapshot, SessionSummary } from "./session-summary.js";
 import type { ContextBudget, SiftLightResult } from "./types.js";
 
@@ -12,7 +12,7 @@ export class SiftLightRuntime {
   }
 
   async search(
-    input: SiftLightInput,
+    input: SiftLightRequest,
     cwd: string,
     signal?: AbortSignal,
     contextBudget?: ContextBudget,

package/src/service.ts

@@ -7,6 +7,18 @@ import { resolve } from "node:path";
 import { CursorError, SiftLightError } from "./errors.js";
 import { DISCOVERY_MODE_REQUIRED_ERROR } from "./discovery-errors.js";
 import { validateRequestContract } from "./request-contract.js";
+import { normalizeRequestAliases, type SiftLightRequest } from "./request-aliases.js";
+
+export type { SiftLightRequest } from "./request-aliases.js";
+
+/** Normalizations are part of the evidence: show them in text and details. */
+function withRequestNotes(result: SiftLightResult, notes: readonly string[]): SiftLightResult {
+  if (notes.length === 0) return result;
+  return {
+    text: `${notes.map((note) => `[Request note: ${note}]`).join("\n")}\n${result.text}`,
+    details: { ...result.details, requestNotes: [...notes] },
+  };
+}
 import {
   formatMatchMetadataPage,
   formatMatchPage,
@@ -392,10 +404,20 @@ export class SiftLightService {
   }
 
   async search(
-    input: SiftLightInput,
+    request: SiftLightRequest,
     cwd: string,
     signal?: AbortSignal,
     options: SiftLightSearchOptions = {},
+  ): Promise<SiftLightResult> {
+    const { input, notes } = normalizeRequestAliases(request);
+    return withRequestNotes(await this.#searchNormalized(input, cwd, signal, options), notes);
+  }
+
+  async #searchNormalized(
+    input: SiftLightInput,
+    cwd: string,
+    signal: AbortSignal | undefined,
+    options: SiftLightSearchOptions,
   ): Promise<SiftLightResult> {
     validateRawSearchInput(input);
     validateRequestContract(input);

package/src/session-summary.ts

@@ -1,6 +1,6 @@
 import type { SiftLightLocale } from "./config.js";
 import { SIFT_LIGHT_VERSION } from "./package-version.js";
-import type { SiftLightInput } from "./service.js";
+import type { SiftLightRequest } from "./service.js";
 import type { SiftLightResult } from "./types.js";
 
 export const SESSION_STATUS_KEY = "sift-light_session";
@@ -13,7 +13,7 @@ export interface SessionSummarySnapshot {
   failedCalls: number;
 }
 
-function isNewQuery(input: SiftLightInput): boolean {
+function isNewQuery(input: SiftLightRequest): boolean {
   return (
     input.cursor === undefined &&
     input.sourceCursor === undefined &&
@@ -21,7 +21,7 @@ function isNewQuery(input: SiftLightInput): boolean {
   );
 }
 
-function wasAutomaticallyOrganized(input: SiftLightInput, result: SiftLightResult): boolean {
+function wasAutomaticallyOrganized(input: SiftLightRequest, result: SiftLightResult): boolean {
   const autoMode = input.mode === undefined || input.mode === "auto";
   return autoMode && input.limit === undefined && result.details.summaryFilesShown !== undefined;
 }
@@ -59,7 +59,7 @@ export class SessionSummary {
     failedCalls: 0,
   };
 
-  record(input: SiftLightInput, result: SiftLightResult): void {
+  record(input: SiftLightRequest, result: SiftLightResult): void {
     if (!isNewQuery(input)) return;
     this.#snapshot.queries += 1;
     if (result.details.status === "complete") this.#snapshot.completeQueries += 1;

package/src/source-inspection.ts

@@ -33,6 +33,8 @@ export interface SourceInspectionTarget {
   /** Raw source byte offset; retained-match focus is line-relative. */
   absoluteFocus?: number;
   focus?: number;
+  /** Structure already proven by the analysis that produced `range`. */
+  structure?: StructureDetails;
   expectedRevision?: SourceRevision;
   unverified?: boolean;
   retry?: SiftLightInput;
@@ -121,7 +123,20 @@ async function prepare(
   let range = target.range;
   let details: StructureDetails = { status: "no-symbol" };
   const language = syntaxLanguage(document.path);
-  if (document.utf8 && language && language !== "go") {
+  if (target.range && target.structure) {
+    document.checkRange(target.range);
+    const lines = {
+      startLine: document.lineAt(target.range.start),
+      endLine: document.lineAt(Math.max(target.range.start, target.range.end - 1)),
+    };
+    // Whole lines keep leading modifiers such as `export` or decorators readable.
+    range = document.lineRange(lines.startLine, lines.endLine);
+    details = {
+      ...target.structure,
+      range: lines,
+      ...(target.structure.symbol ? { symbol: { ...target.structure.symbol, range: lines } } : {}),
+    };
+  } else if (document.utf8 && language && language !== "go") {
     const syntax = await access.syntax(document);
     details = {
       status:
@@ -205,7 +220,9 @@ async function prepare(
     Math.min(document.lineStarts.length, target.line + 10),
   );
   const boundary: Exclude<SourceBoundary, "mixed"> = target.range
-    ? "requested-range"
+    ? target.structure
+      ? "syntax"
+      : "requested-range"
     : details.status === "available" && details.range
       ? "syntax"
       : "line-window";
@@ -692,8 +709,16 @@ export async function continueSource(
   cursor: string,
   access: SourceAccess,
   continuations: SourceContinuations,
+  expectedPath?: string,
 ): Promise<SiftLightResult> {
   const state = continuations.resolve(cursor);
+  if (
+    expectedPath !== undefined &&
+    resolve(access.cwd, expectedPath.replace(/^@/, "")) !== resolve(access.cwd, state.source.path)
+  )
+    throw new SiftLightError(
+      `sourceCursor continues ${JSON.stringify(state.source.path)}, not ${JSON.stringify(expectedPath)}; copy the returned nextRequest exactly`,
+    );
   const document = await access.load(state.source.path, state.source);
   const page = sourcePage(document, state.remaining, MAX_RESULT_BYTES - 1400);
   const next = continuations.advance(cursor, page.fragment);