sift-light 1.0.1 → 1.0.2

package/plugins/sift-light/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sift-light",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "private": true,
   "type": "module",
   "omp": {

package/src/analysis-limits.ts

@@ -13,7 +13,7 @@ export const MAX_ANALYSIS_STORAGE_BYTES = 32 * 1024 * 1024;
 export const ANALYSIS_METADATA_RESERVE_BYTES = 64 * 1024;
 export const MAX_ANALYSIS_REASONS = 64;
 export const MAX_ANALYSIS_REASON_BYTES = 4 * 1024;
-export const MIN_ANY_OF_TERMS = 2;
+export const MIN_ANY_OF_TERMS = 1;
 export const MAX_ANY_OF_TERMS = 8;
 /** Public union size; execution keeps the proven eight-term scan boundary per chunk. */
 export const MAX_ANY_OF_TOTAL_TERMS = 64;

package/src/analysis-store.ts

@@ -206,7 +206,10 @@ function publicAnalysisItem(
   return {
     path: item.path,
     line: item.line,
-    label: result.kind === "outline" && modelOutput ? item.label : publicAnalysisLabel(result),
+    label:
+      (result.kind === "outline" && modelOutput) || result.kind === "files"
+        ? item.label
+        : publicAnalysisLabel(result),
     index: index + 1,
     ...(inspect ? { inspect } : {}),
     ...(publicDetails ? { details: publicDetails } : {}),
@@ -455,7 +458,8 @@ export class AnalysisStore {
         if (items.length >= 30 || !appendItem(index)) break;
       }
     } else {
-      for (let index = offset; index < result.items.length && items.length < 30; index += 1) {
+      const pageSize = result.pageSize ?? 30;
+      for (let index = offset; index < result.items.length && items.length < pageSize; index += 1) {
         if (!appendItem(index)) break;
       }
     }

package/src/analysis-types.ts

@@ -4,7 +4,12 @@ import type { SearchScopeDetails, ResultStatistics } from "./types.js";
 import type { ValidationDetails } from "./validation-types.js";
 import type { ConceptSourceSummary } from "./concept-source-generation.js";
 
-export type CoverageStatus = "complete" | "partial" | "skipped" | "not-applicable";
+export type CoverageStatus =
+  | "complete"
+  | "partial"
+  | "policy-filtered"
+  | "skipped"
+  | "not-applicable";
 
 export const SEMANTIC_JUDGE_CLASSIFICATIONS = [
   "implementation-candidate",
@@ -125,6 +130,8 @@ export interface AnalysisDetails {
     passagesRanked?: number;
     elapsedMs?: number;
     filesEnumerated?: number;
+    ignoredFiles?: number;
+    ignoredMatches?: number;
     filesAdmitted?: number;
     filesParsed?: number;
     filesSkipped?: number;
@@ -159,4 +166,6 @@ export interface AnalysisResultSet {
   stats?: AnalysisDetails["stats"];
   sourceGeneration?: ConceptSourceSummary;
   redact?: boolean;
+  /** Explicit output page size (`limit`); pages default to 30 items. */
+  pageSize?: number;
 }

package/src/evidence-service.ts

@@ -39,7 +39,6 @@ import {
   continueSource,
   inspectDocuments,
   matchInspectionTarget,
-  inspectDocumentsMetadata,
   type SourceInspectionTarget,
 } from "./source-inspection.js";
 import { validateSavedEvidence } from "./evidence-validation.js";
@@ -48,6 +47,7 @@ import type { CodeStructureProvider } from "./structure.js";
 import { filterRoleOccurrences, findFunctionConjunctions } from "./syntax-search.js";
 import { syntaxLanguage } from "./syntax.js";
 import { parsePythonOutline } from "./python-outline.js";
+import { pythonRoleAnalysis } from "./python-roles.js";
 import {
   combineHybridSearch,
   hybridConceptLimit,
@@ -66,10 +66,12 @@ import {
   MAX_STRUCTURE_FILES,
 } from "./analysis-limits.js";
 import {
+  MAX_INSPECT_REQUEST_TARGETS,
   MAX_INSPECT_TARGETS,
   type SearchRequest,
   type SearchScopeDetails,
   type SiftLightResult,
+  type StructureDetails,
 } from "./types.js";
 
 export function isEvidenceRequest(input: SiftLightInput): boolean {
@@ -95,6 +97,100 @@ export function isEvidenceRequest(input: SiftLightInput): boolean {
   );
 }
 
+/**
+ * Splits one inspect request into the page served now and an exact request for
+ * the rest. `paths` without a cursor opens each file from line 1.
+ */
+function pageInspectRequest(input: SiftLightInput): {
+  current: SiftLightInput;
+  remaining?: SiftLightInput;
+} {
+  if (input.scope === "expand")
+    throw new SiftLightError(
+      "mode=inspect reads exact locations; scope=expand does not apply (omit scope)",
+    );
+  const { scope: _scope, paths, ...rest } = input;
+  let request: SiftLightInput = rest;
+  if (paths !== undefined) {
+    if (
+      input.cursor !== undefined ||
+      input.targets !== undefined ||
+      input.matchIndices !== undefined ||
+      input.path !== undefined ||
+      input.line !== undefined ||
+      input.matchIndex !== undefined
+    )
+      throw new SiftLightError(
+        "mode=inspect paths opens files from line 1 and cannot be combined with cursor, targets, matchIndices, path, line or matchIndex",
+      );
+    request = { ...rest, targets: paths.map((path) => ({ path, line: 1 })) };
+  }
+  const redact = input.redact ? { redact: true } : {};
+  const requested = request.targets?.length ?? request.matchIndices?.length ?? 0;
+  if (requested > MAX_INSPECT_REQUEST_TARGETS)
+    throw new SiftLightError(
+      `mode=inspect accepts at most ${String(MAX_INSPECT_REQUEST_TARGETS)} targets per request; ${String(MAX_INSPECT_TARGETS)} are inspected per response`,
+    );
+  if (request.targets && request.targets.length > MAX_INSPECT_TARGETS)
+    return {
+      current: { ...request, targets: request.targets.slice(0, MAX_INSPECT_TARGETS) },
+      remaining: {
+        mode: "inspect",
+        targets: request.targets.slice(MAX_INSPECT_TARGETS),
+        ...redact,
+      },
+    };
+  if (request.matchIndices && request.matchIndices.length > MAX_INSPECT_TARGETS)
+    return {
+      current: { ...request, matchIndices: request.matchIndices.slice(0, MAX_INSPECT_TARGETS) },
+      remaining: {
+        mode: "inspect",
+        ...(request.cursor !== undefined ? { cursor: request.cursor } : {}),
+        matchIndices: request.matchIndices.slice(MAX_INSPECT_TARGETS),
+        ...redact,
+      },
+    };
+  return { current: request };
+}
+
+/** The unserved targets stay explicit: the page is partial and names its exact continuation. */
+function withRemainingInspection(
+  result: SiftLightResult,
+  remaining: SiftLightInput,
+): SiftLightResult {
+  const count = remaining.targets?.length ?? remaining.matchIndices?.length ?? 0;
+  return {
+    text: `${result.text}\n\n[${String(count)} more target(s) were not inspected; one response inspects ${String(MAX_INSPECT_TARGETS)}.]\nNext request: ${JSON.stringify(remaining)}`,
+    details: {
+      ...result.details,
+      status: "partial",
+      snapshotComplete: false,
+      nextRequest: remaining,
+    },
+  };
+}
+
+/** Structure facts already proven by the analysis that produced a bounded item. */
+function analysisItemStructure(item: AnalysisItem): StructureDetails {
+  const details = item.details ?? {};
+  const language = typeof details.language === "string" ? details.language : undefined;
+  const name = typeof details.name === "string" ? details.name : undefined;
+  const scope = Array.isArray(details.scope)
+    ? details.scope.filter((entry): entry is string => typeof entry === "string")
+    : typeof details.scope === "string"
+      ? [details.scope]
+      : [];
+  const kind = details.kind === "function" ? "function" : (item.label.split(" ", 1)[0] ?? "symbol");
+  return {
+    status: "available",
+    provider: language === "python" ? "python-outline" : "tree-sitter",
+    ...(language ? { language } : {}),
+    ...(name
+      ? { symbol: { name, kind, scope, range: { startLine: item.line, endLine: item.line } } }
+      : {}),
+  };
+}
+
 export interface EvidenceSearchOptions {
   modelOutput?: boolean;
   onProgress?: (progress: OperationProgress) => void;
@@ -126,12 +222,12 @@ function validateTerms(input: SiftLightInput): string[] | undefined {
   }
   if (
     !Array.isArray(terms) ||
-    terms.length < 2 ||
+    terms.length < 1 ||
     terms.length > 3 ||
     terms.some((term) => typeof term !== "string" || !term.trim() || /[\r\n\0]/.test(term)) ||
     new Set(terms).size !== terms.length
   )
-    throw new SiftLightError("allOf requires 2–3 distinct, nonempty, single-line literal terms");
+    throw new SiftLightError("allOf requires 1–3 distinct, nonempty, single-line literal terms");
   if (
     input.pattern !== undefined ||
     input.roles !== undefined ||
@@ -396,13 +492,13 @@ export class EvidenceService {
       if (typeof input.sourceCursor !== "string" || !input.sourceCursor.trim())
         throw new CursorError("A nonempty sourceCursor is required");
       if (input.mode !== "inspect") throw new SiftLightError("sourceCursor requires mode=inspect");
-      return continueSource(input.sourceCursor, access, this.#continuations);
+      return continueSource(input.sourceCursor, access, this.#continuations, input.path);
     }
     if (input.mode === "inspect") {
-      const targets = this.#inspectionTargets(input, cwd);
-      return targets.some((target) => target.range !== undefined)
-        ? inspectDocumentsMetadata(targets, access, this.#structure)
-        : inspectDocuments(targets, access, this.#continuations, this.#structure);
+      const { current, remaining } = pageInspectRequest(input);
+      const targets = this.#inspectionTargets(current, cwd);
+      const result = await inspectDocuments(targets, access, this.#continuations, this.#structure);
+      return remaining ? withRemainingInspection(result, remaining) : result;
     }
     if (input.mode === "concept") {
       const execution = await this.#conceptSearch(input, access, options.onProgress);
@@ -518,7 +614,7 @@ export class EvidenceService {
           "anyOf is an explicit case-sensitive literal union; omit pattern, allOf, within, roles, literal and ignoreCase",
         );
       if (input.mode !== undefined && input.mode !== "auto" && input.mode !== "matches")
-        throw new SiftLightError("anyOf mode must be omitted, auto, or matches");
+        throw new SiftLightError("anyOf mode must be omitted, auto, matches or summary");
       const chunks = Array.from(
         { length: Math.ceil(anyOf.length / MAX_ANY_OF_TERMS) },
         (_, index) => anyOf.slice(index * MAX_ANY_OF_TERMS, (index + 1) * MAX_ANY_OF_TERMS),
@@ -696,8 +792,12 @@ export class EvidenceService {
           );
           if (item) result.items.push(item);
         } else if (terms || input.roles) {
-          if (syntaxLanguage(file.document.path)) syntaxCapableFiles += 1;
-          const syntax = await access.syntax(file.document);
+          // Python roles come from a bounded lexical scanner; other languages parse.
+          const pythonRoles = !terms && /\.py$/iu.test(file.document.path);
+          if (pythonRoles || syntaxLanguage(file.document.path)) syntaxCapableFiles += 1;
+          const syntax = pythonRoles
+            ? pythonRoleAnalysis(file.document)
+            : await access.syntax(file.document);
           const classified = terms
             ? findFunctionConjunctions(
                 file.document,
@@ -790,12 +890,16 @@ export class EvidenceService {
       const item = this.#analyses.item(input.cursor, input.matchIndex);
       if (!item.source || !item.range)
         throw new CursorError("This analysis item has no verified source range");
-      const metadataOnly = item.details?.kind === "symbol" || item.details?.kind === "function";
+      // Outline symbols and function conjunctions carry their exact syntax range,
+      // so inspection returns that version-checked range as source (issue #96).
+      const bounded = item.details?.kind === "symbol" || item.details?.kind === "function";
       return {
         path: item.path,
         line: item.line,
         reference: item.source,
-        ...(metadataOnly ? { range: item.range } : { absoluteFocus: item.range.start }),
+        ...(bounded
+          ? { range: item.range, structure: analysisItemStructure(item) }
+          : { absoluteFocus: item.range.start }),
       };
     }
     return {

package/src/file-discovery.ts

@@ -13,6 +13,42 @@ interface FileScore {
 const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
 /** Bracket classes remain valid query text, so dynamic route names stay searchable. */
 const FILE_QUERY_GLOB_WILDCARD = /[*?]/u;
+/** Queries such as `*`, `**` or `.*` mean "every file" rather than filename text. */
+const FILE_QUERY_MATCH_ALL = /^(?:[*/]+|\.\*|\.\+|\*\.\*)$/u;
+/** Regex syntax that cannot be a simple glob; such a query is rejected with guidance. */
+const FILE_QUERY_REGEX_SYNTAX = /[()|^$\\+]/u;
+const IGNORED_MATCH_SAMPLES = 5;
+
+interface NormalizedFileQuery {
+  query: string;
+  glob?: string;
+  note?: string;
+}
+
+/**
+ * Filename discovery matches text. A wildcard-only query means "all files" and a
+ * plain glob such as `*.py` becomes an explicit glob filter; both are disclosed.
+ */
+function normalizeFileQuery(query: string, hasGlob: boolean): NormalizedFileQuery {
+  const trimmed = query.trim();
+  if (FILE_QUERY_MATCH_ALL.test(trimmed))
+    return {
+      query: "",
+      note: `Query ${JSON.stringify(query)} was treated as listing every file under path; omit query for the same result.`,
+    };
+  if (!FILE_QUERY_GLOB_WILDCARD.test(trimmed)) return { query };
+  if (!hasGlob && !FILE_QUERY_REGEX_SYNTAX.test(trimmed) && !/\s/u.test(trimmed)) {
+    const glob = trimmed.includes("/") ? trimmed : `**/${trimmed}`;
+    return {
+      query: "",
+      glob,
+      note: `Query ${JSON.stringify(query)} contains glob wildcards and was applied as glob ${JSON.stringify(glob)}; mode=files query otherwise matches filename text.`,
+    };
+  }
+  throw new SiftLightError(
+    `File query ${JSON.stringify(query)} uses wildcard or regex syntax; mode=files matches filename and path text, not patterns. Omit query to retain every file under path, use glob for one name pattern, or run one files request per name.`,
+  );
+}
 
 function subsequenceScore(text: string, query: string): number | undefined {
   const characters = Array.from(graphemes.segment(text), (item) => item.segment);
@@ -70,26 +106,33 @@ export async function discoverFiles(
   cwd: string,
   signal?: AbortSignal,
 ): Promise<AnalysisResultSet> {
-  const query = input.query ?? "";
-  if (query.length > 256 || !query.isWellFormed() || /[\r\n\0]/.test(query))
+  const rawQuery = input.query ?? "";
+  if (rawQuery.length > 256 || !rawQuery.isWellFormed() || /[\r\n\0]/.test(rawQuery))
     throw new SiftLightError(
       "File query must be well-formed single-line text of at most 256 characters",
     );
-  // A wildcard query can never match filename text, so scoring it would discard
-  // every enumerated file and report an empty result as complete.
-  if (FILE_QUERY_GLOB_WILDCARD.test(query))
-    throw new SiftLightError(
-      `File query ${JSON.stringify(query)} uses glob wildcards; mode=files matches filename and path text, not glob patterns. Omit query to retain every file under path, or use glob to filter by name pattern.`,
-    );
-  const request = normalizeRequest({ ...input, pattern: "" });
+  const requestedGlobs =
+    input.glob === undefined ? [] : Array.isArray(input.glob) ? input.glob : [input.glob];
+  const normalizedQuery = normalizeFileQuery(rawQuery, requestedGlobs.length > 0);
+  const query = normalizedQuery.query;
+  const request = normalizeRequest({
+    ...input,
+    pattern: "",
+    ...(normalizedQuery.glob ? { glob: [...requestedGlobs, normalizedQuery.glob] } : {}),
+  });
   const policy = new SearchPathPolicy(cwd);
   const discoveryRoot = request.path ?? ".";
   const scoringRoot = await policy.resolveSearchTarget(discoveryRoot);
-  const files = await listWorkspaceFiles(cwd, signal, {
+  const includeIgnored = request.ignorePolicy === "include";
+  const listOptions = {
     path: scoringRoot,
     glob: request.glob,
     exclude: request.exclude,
     hidden: request.hidden,
+  };
+  const files = await listWorkspaceFiles(cwd, signal, {
+    ...listOptions,
+    ...(includeIgnored ? { ignore: false, ignoreParents: false } : {}),
   });
   const filtered = await filterPathsByModificationTime(
     cwd,
@@ -98,10 +141,12 @@ export async function discoverFiles(
     request.modifiedBeforeMs,
     signal,
   );
+  const rank = (path: string) =>
+    scoreFilePath(pathRelativeToDiscoveryRoot(cwd, scoringRoot, path), query);
   const selected = filtered.paths
     .flatMap((path) => {
-      const rank = scoreFilePath(pathRelativeToDiscoveryRoot(cwd, scoringRoot, path), query);
-      return rank ? [{ path, ...rank }] : [];
+      const score = rank(path);
+      return score ? [{ path, ...score }] : [];
     })
     .toSorted((left, right) => right.score - left.score || left.path.localeCompare(right.path));
   for (let offset = 0; offset < selected.length; offset += 16) {
@@ -111,15 +156,54 @@ export async function discoverFiles(
     );
     signal?.throwIfAborted();
   }
+  const reasons = new Set([...files.reasons, ...filtered.reasons]);
+  if (normalizedQuery.note) reasons.add(normalizedQuery.note);
+  // Issue #97: ignore rules must never turn a filtered absence into a complete one.
+  let ignoredFiles = 0;
+  let ignoredMatches = 0;
+  let ignoredComparisonPartial = false;
+  if (!includeIgnored) {
+    const everything = await listWorkspaceFiles(cwd, signal, {
+      ...listOptions,
+      ignore: false,
+      ignoreParents: false,
+    });
+    ignoredComparisonPartial = everything.partial;
+    const admitted = new Set(files.paths);
+    const ignored = everything.paths.filter((path) => !admitted.has(path));
+    ignoredFiles = ignored.length;
+    const ignoredCandidates = ignored
+      .flatMap((path) => {
+        const score = rank(path);
+        return score ? [{ path, ...score }] : [];
+      })
+      .toSorted((left, right) => right.score - left.score || left.path.localeCompare(right.path));
+    ignoredMatches = ignoredCandidates.length;
+    if (ignoredFiles > 0) {
+      const samples = ignoredCandidates
+        .slice(0, IGNORED_MATCH_SAMPLES)
+        .map((item) => `${JSON.stringify(item.path)} (${item.reason})`);
+      reasons.add(
+        ignoredMatches > 0
+          ? `${String(ignoredFiles)} file(s) were excluded by ignore rules; ${String(ignoredMatches)} of them match this query${samples.length ? `, e.g. ${samples.join(", ")}` : ""}. Retry with ignorePolicy "include" to list them.`
+          : `${String(ignoredFiles)} file(s) were excluded by ignore rules; none of them match this query.`,
+      );
+    }
+    if (ignoredComparisonPartial)
+      reasons.add(
+        "Ignored-file comparison was incomplete; the ignored-file count is a lower bound",
+      );
+  }
+  const enumerationPartial = files.partial || filtered.partial;
   return {
     kind: "files",
     unit: "files",
-    partial: files.partial || filtered.partial,
-    reasons: [...new Set([...files.reasons, ...filtered.reasons])],
+    partial: enumerationPartial,
+    reasons: [...reasons],
     items: selected.map((item) => ({
       path: item.path,
       line: 1,
-      label: `File candidate (${item.reason})`,
+      label: `File candidate (score ${String(item.score)}: ${item.reason})`,
       details: {
         kind: "file",
         score: item.score,
@@ -127,8 +211,17 @@ export async function discoverFiles(
         inspect: { mode: "inspect", path: item.path, line: 1 },
       },
     })),
-    coverage: { fileEnumeration: files.partial || filtered.partial ? "partial" : "complete" },
-    stats: { filesEnumerated: files.paths.length },
+    coverage: {
+      fileEnumeration: enumerationPartial
+        ? "partial"
+        : ignoredFiles > 0 || ignoredComparisonPartial
+          ? "policy-filtered"
+          : "complete",
+    },
+    stats: {
+      filesEnumerated: files.paths.length,
+      ...(includeIgnored ? {} : { ignoredFiles, ignoredMatches }),
+    },
     scope: {
       path: request.path ?? ".",
       requestedPath: request.path ?? ".",
@@ -146,5 +239,6 @@ export async function discoverFiles(
         : {}),
     },
     redact: input.redact ?? false,
+    ...(input.limit !== undefined ? { pageSize: request.pageSize } : {}),
   };
 }

package/src/inspect.ts

@@ -78,6 +78,9 @@ export function resolveInspectionTarget(
     line = retainedMatch.lineNumber;
   }
   if (!path) throw new SiftLightError("path is required when mode=inspect");
+  // A direct path without a line opens the file from its first line. Cursor
+  // inspection still needs the exact retained line, so it keeps failing closed.
+  if (line === undefined && input.cursor === undefined) line = 1;
   if (line === undefined || !Number.isSafeInteger(line) || line < 1) {
     throw new SiftLightError("line must be a positive integer when mode=inspect");
   }

package/src/language-capability-definitions.ts

@@ -119,6 +119,15 @@ const PYTHON_LANGUAGE_CAPABILITIES: readonly LanguageCapabilitySpec[] = [
     availability: "implemented",
     load: "lazy",
   },
+  {
+    id: "python-lexical.roles",
+    name: "roles",
+    provider: "bounded Python lexical role scanner",
+    providerKind: "builtin",
+    evidence: "syntax",
+    availability: "implemented",
+    load: "lazy",
+  },
 ];
 
 export const DEFAULT_LANGUAGE_CAPABILITIES: readonly LanguageCapabilityDescriptor[] = [

package/src/mcp-model-output.ts

@@ -87,7 +87,10 @@ function compactRows(analysis: AnalysisDetails): string[] {
       rows.push(JSON.stringify(item.path));
       previousPath = item.path;
     }
-    const label = analysis.kind === "outline" && analysis.modelOutput ? item.label : "metadata";
+    const label =
+      (analysis.kind === "outline" && analysis.modelOutput) || analysis.kind === "files"
+        ? item.label
+        : "metadata";
     const semanticJudge = item.details?.semanticJudge;
     const judgment =
       isRecord(semanticJudge) &&
@@ -149,6 +152,7 @@ export function compactMcpModelText(result: SiftLightResult): string {
   const inspect = compactInspectInstruction(analysis);
   const nextRequest = distinctNextRequest(result.details, analysis);
   const compact = [
+    ...(result.details.requestNotes ?? []).map((note) => `[Request note: ${note}]`),
     header,
     ...compactMetadata(result.details, analysis),
     ...compactRows(analysis),
@@ -156,6 +160,7 @@ export function compactMcpModelText(result: SiftLightResult): string {
     ...(nextRequest ? [`Next request: ${nextRequest}`] : []),
   ].join("\n");
   const standard = result.text.replace(" Structured output retains per-item evidence details.", "");
+  // Both projections carry request notes, so the byte comparison stays fair.
   if (analysis.semanticJudge) return compact;
   return Buffer.byteLength(compact) < Buffer.byteLength(standard) ? compact : standard;
 }