sift-light 1.0.0 → 1.0.2

package/src/file-discovery.ts

@@ -13,6 +13,42 @@ interface FileScore {
 const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
 /** Bracket classes remain valid query text, so dynamic route names stay searchable. */
 const FILE_QUERY_GLOB_WILDCARD = /[*?]/u;
+/** Queries such as `*`, `**` or `.*` mean "every file" rather than filename text. */
+const FILE_QUERY_MATCH_ALL = /^(?:[*/]+|\.\*|\.\+|\*\.\*)$/u;
+/** Regex syntax that cannot be a simple glob; such a query is rejected with guidance. */
+const FILE_QUERY_REGEX_SYNTAX = /[()|^$\\+]/u;
+const IGNORED_MATCH_SAMPLES = 5;
+
+interface NormalizedFileQuery {
+  query: string;
+  glob?: string;
+  note?: string;
+}
+
+/**
+ * Filename discovery matches text. A wildcard-only query means "all files" and a
+ * plain glob such as `*.py` becomes an explicit glob filter; both are disclosed.
+ */
+function normalizeFileQuery(query: string, hasGlob: boolean): NormalizedFileQuery {
+  const trimmed = query.trim();
+  if (FILE_QUERY_MATCH_ALL.test(trimmed))
+    return {
+      query: "",
+      note: `Query ${JSON.stringify(query)} was treated as listing every file under path; omit query for the same result.`,
+    };
+  if (!FILE_QUERY_GLOB_WILDCARD.test(trimmed)) return { query };
+  if (!hasGlob && !FILE_QUERY_REGEX_SYNTAX.test(trimmed) && !/\s/u.test(trimmed)) {
+    const glob = trimmed.includes("/") ? trimmed : `**/${trimmed}`;
+    return {
+      query: "",
+      glob,
+      note: `Query ${JSON.stringify(query)} contains glob wildcards and was applied as glob ${JSON.stringify(glob)}; mode=files query otherwise matches filename text.`,
+    };
+  }
+  throw new SiftLightError(
+    `File query ${JSON.stringify(query)} uses wildcard or regex syntax; mode=files matches filename and path text, not patterns. Omit query to retain every file under path, use glob for one name pattern, or run one files request per name.`,
+  );
+}
 
 function subsequenceScore(text: string, query: string): number | undefined {
   const characters = Array.from(graphemes.segment(text), (item) => item.segment);
@@ -70,26 +106,33 @@ export async function discoverFiles(
   cwd: string,
   signal?: AbortSignal,
 ): Promise<AnalysisResultSet> {
-  const query = input.query ?? "";
-  if (query.length > 256 || !query.isWellFormed() || /[\r\n\0]/.test(query))
+  const rawQuery = input.query ?? "";
+  if (rawQuery.length > 256 || !rawQuery.isWellFormed() || /[\r\n\0]/.test(rawQuery))
     throw new SiftLightError(
       "File query must be well-formed single-line text of at most 256 characters",
     );
-  // A wildcard query can never match filename text, so scoring it would discard
-  // every enumerated file and report an empty result as complete.
-  if (FILE_QUERY_GLOB_WILDCARD.test(query))
-    throw new SiftLightError(
-      `File query ${JSON.stringify(query)} uses glob wildcards; mode=files matches filename and path text, not glob patterns. Omit query to retain every file under path, or use glob to filter by name pattern.`,
-    );
-  const request = normalizeRequest({ ...input, pattern: "" });
+  const requestedGlobs =
+    input.glob === undefined ? [] : Array.isArray(input.glob) ? input.glob : [input.glob];
+  const normalizedQuery = normalizeFileQuery(rawQuery, requestedGlobs.length > 0);
+  const query = normalizedQuery.query;
+  const request = normalizeRequest({
+    ...input,
+    pattern: "",
+    ...(normalizedQuery.glob ? { glob: [...requestedGlobs, normalizedQuery.glob] } : {}),
+  });
   const policy = new SearchPathPolicy(cwd);
   const discoveryRoot = request.path ?? ".";
   const scoringRoot = await policy.resolveSearchTarget(discoveryRoot);
-  const files = await listWorkspaceFiles(cwd, signal, {
+  const includeIgnored = request.ignorePolicy === "include";
+  const listOptions = {
     path: scoringRoot,
     glob: request.glob,
     exclude: request.exclude,
     hidden: request.hidden,
+  };
+  const files = await listWorkspaceFiles(cwd, signal, {
+    ...listOptions,
+    ...(includeIgnored ? { ignore: false, ignoreParents: false } : {}),
   });
   const filtered = await filterPathsByModificationTime(
     cwd,
@@ -98,10 +141,12 @@ export async function discoverFiles(
     request.modifiedBeforeMs,
     signal,
   );
+  const rank = (path: string) =>
+    scoreFilePath(pathRelativeToDiscoveryRoot(cwd, scoringRoot, path), query);
   const selected = filtered.paths
     .flatMap((path) => {
-      const rank = scoreFilePath(pathRelativeToDiscoveryRoot(cwd, scoringRoot, path), query);
-      return rank ? [{ path, ...rank }] : [];
+      const score = rank(path);
+      return score ? [{ path, ...score }] : [];
     })
     .toSorted((left, right) => right.score - left.score || left.path.localeCompare(right.path));
   for (let offset = 0; offset < selected.length; offset += 16) {
@@ -111,15 +156,54 @@ export async function discoverFiles(
     );
     signal?.throwIfAborted();
   }
+  const reasons = new Set([...files.reasons, ...filtered.reasons]);
+  if (normalizedQuery.note) reasons.add(normalizedQuery.note);
+  // Issue #97: ignore rules must never turn a filtered absence into a complete one.
+  let ignoredFiles = 0;
+  let ignoredMatches = 0;
+  let ignoredComparisonPartial = false;
+  if (!includeIgnored) {
+    const everything = await listWorkspaceFiles(cwd, signal, {
+      ...listOptions,
+      ignore: false,
+      ignoreParents: false,
+    });
+    ignoredComparisonPartial = everything.partial;
+    const admitted = new Set(files.paths);
+    const ignored = everything.paths.filter((path) => !admitted.has(path));
+    ignoredFiles = ignored.length;
+    const ignoredCandidates = ignored
+      .flatMap((path) => {
+        const score = rank(path);
+        return score ? [{ path, ...score }] : [];
+      })
+      .toSorted((left, right) => right.score - left.score || left.path.localeCompare(right.path));
+    ignoredMatches = ignoredCandidates.length;
+    if (ignoredFiles > 0) {
+      const samples = ignoredCandidates
+        .slice(0, IGNORED_MATCH_SAMPLES)
+        .map((item) => `${JSON.stringify(item.path)} (${item.reason})`);
+      reasons.add(
+        ignoredMatches > 0
+          ? `${String(ignoredFiles)} file(s) were excluded by ignore rules; ${String(ignoredMatches)} of them match this query${samples.length ? `, e.g. ${samples.join(", ")}` : ""}. Retry with ignorePolicy "include" to list them.`
+          : `${String(ignoredFiles)} file(s) were excluded by ignore rules; none of them match this query.`,
+      );
+    }
+    if (ignoredComparisonPartial)
+      reasons.add(
+        "Ignored-file comparison was incomplete; the ignored-file count is a lower bound",
+      );
+  }
+  const enumerationPartial = files.partial || filtered.partial;
   return {
     kind: "files",
     unit: "files",
-    partial: files.partial || filtered.partial,
-    reasons: [...new Set([...files.reasons, ...filtered.reasons])],
+    partial: enumerationPartial,
+    reasons: [...reasons],
     items: selected.map((item) => ({
       path: item.path,
       line: 1,
-      label: `File candidate (${item.reason})`,
+      label: `File candidate (score ${String(item.score)}: ${item.reason})`,
       details: {
         kind: "file",
         score: item.score,
@@ -127,8 +211,17 @@ export async function discoverFiles(
         inspect: { mode: "inspect", path: item.path, line: 1 },
       },
     })),
-    coverage: { fileEnumeration: files.partial || filtered.partial ? "partial" : "complete" },
-    stats: { filesEnumerated: files.paths.length },
+    coverage: {
+      fileEnumeration: enumerationPartial
+        ? "partial"
+        : ignoredFiles > 0 || ignoredComparisonPartial
+          ? "policy-filtered"
+          : "complete",
+    },
+    stats: {
+      filesEnumerated: files.paths.length,
+      ...(includeIgnored ? {} : { ignoredFiles, ignoredMatches }),
+    },
     scope: {
       path: request.path ?? ".",
       requestedPath: request.path ?? ".",
@@ -146,5 +239,6 @@ export async function discoverFiles(
         : {}),
     },
     redact: input.redact ?? false,
+    ...(input.limit !== undefined ? { pageSize: request.pageSize } : {}),
   };
 }

package/src/hybrid-search.ts

@@ -274,6 +274,14 @@ export async function combineHybridSearch(
   const selectionReason = conceptCandidatesOmitted
     ? `Hybrid concept limit retained the top ${String(selectedConcept.length)} of ${String(eligibleConcept.length)} non-overlapping semantic candidates`
     : undefined;
+  const firstScore = Number(
+    eligibleConcept[0]?.details?.rankingScore ?? eligibleConcept[0]?.details?.score,
+  );
+  const secondScore = Number(
+    eligibleConcept[1]?.details?.rankingScore ?? eligibleConcept[1]?.details?.score,
+  );
+  const closeRanking =
+    Number.isFinite(firstScore) && Number.isFinite(secondScore) && firstScore - secondScore < 0.01;
   return {
     kind: "hybrid",
     unit: "evidence-items",
@@ -285,6 +293,11 @@ export async function combineHybridSearch(
       ...literal.reasons,
       ...execution.sourceGeneration.reasons,
       ...(selectionReason ? [selectionReason] : []),
+      ...(closeRanking
+        ? [
+            "Semantic ranks are close; verify the leading candidates with source inspection or literal terms.",
+          ]
+        : []),
       ...(judgedConcept.semanticJudge?.reason ? [judgedConcept.semanticJudge.reason] : []),
     ],
     filesRead: (concept.filesRead ?? 0) + access.filesRead,

package/src/index.ts

@@ -1,11 +1,6 @@
 import { SIFT_LIGHT_DESCRIPTION, siftLightSchema } from "./tool-schema.js";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import {
-  normalizeSearchEnforcement,
-  readSiftLightConfig,
-  DEFAULT_SEMANTIC_JUDGE_CONFIG,
-  type SiftLightConfig,
-} from "./config.js";
+import { normalizeSearchEnforcement, readSiftLightConfig, type SiftLightConfig } from "./config.js";
 import { resolveContextBudget } from "./context-budget.js";
 import { createRipgrepRunner } from "./rg.js";
 import { createCtagsStructureProvider } from "./structure.js";
@@ -17,7 +12,7 @@ import type { SiftLightDetails } from "./types.js";
 import { renderSiftLightCall, renderSiftLightResult } from "./tui/renderers.js";
 import { registerPiSearchPolicy } from "./pi-search-policy.js";
 import { modelErrorText } from "./model-error.js";
-import { createSemanticJudgeIntegration } from "./semantic-judge.js";
+import { createConfiguredSemanticJudgeIntegration } from "./semantic-judge.js";
 
 const SIFT_LIGHT_LABEL = "sift-light";
 
@@ -27,13 +22,12 @@ export async function registerSiftLightExtension(
   pi: ExtensionAPI,
   config: SiftLightConfig,
 ): Promise<void> {
-  const semanticJudge = createSemanticJudgeIntegration(
-    config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG,
-  );
+  const semanticJudge = createConfiguredSemanticJudgeIntegration(config);
   const runtime = new SiftLightRuntime(
     new SiftLightService({
       runRipgrep: createRipgrepRunner(),
       structure: createCtagsStructureProvider(),
+      vectorSearchEnabled: config.vectorSearchEnabled === true,
       semanticJudge,
     }),
   );

package/src/inspect.ts

@@ -78,6 +78,9 @@ export function resolveInspectionTarget(
     line = retainedMatch.lineNumber;
   }
   if (!path) throw new SiftLightError("path is required when mode=inspect");
+  // A direct path without a line opens the file from its first line. Cursor
+  // inspection still needs the exact retained line, so it keeps failing closed.
+  if (line === undefined && input.cursor === undefined) line = 1;
   if (line === undefined || !Number.isSafeInteger(line) || line < 1) {
     throw new SiftLightError("line must be a positive integer when mode=inspect");
   }

package/src/language-capability-definitions.ts

@@ -119,6 +119,15 @@ const PYTHON_LANGUAGE_CAPABILITIES: readonly LanguageCapabilitySpec[] = [
     availability: "implemented",
     load: "lazy",
   },
+  {
+    id: "python-lexical.roles",
+    name: "roles",
+    provider: "bounded Python lexical role scanner",
+    providerKind: "builtin",
+    evidence: "syntax",
+    availability: "implemented",
+    load: "lazy",
+  },
 ];
 
 export const DEFAULT_LANGUAGE_CAPABILITIES: readonly LanguageCapabilityDescriptor[] = [

package/src/mcp-model-output.ts

@@ -7,7 +7,48 @@ function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+function compactSemanticMetadata(details: SiftLightDetails, analysis: AnalysisDetails): string[] {
+  const counts = analysis.counts;
+  const stats = analysis.stats;
+  const judge = analysis.semanticJudge;
+  const coverage = analysis.coverage;
+  return [
+    ...(counts || stats
+      ? [
+          `Search: ${String(counts?.filesAdmitted ?? stats?.filesAdmitted ?? 0)} files, ${String(stats?.passagesRanked ?? counts?.passagesQueued ?? 0)} passages; ${String(stats?.elapsedMs ?? 0)} ms; peak inference RSS ${String(stats?.inferencePeakRssBytes ?? 0)} bytes; cache ${String(stats?.conceptCacheHits ?? 0)} hits/${String(stats?.conceptCacheMisses ?? 0)} misses.`,
+        ]
+      : []),
+    ...(coverage
+      ? [
+          `Coverage: ${Object.entries(coverage)
+            .map(([name, status]) => `${name}=${status}`)
+            .join(", ")}.`,
+        ]
+      : []),
+    ...(analysis.scope
+      ? [`Scope: ${analysis.scope.path}; ignore=${analysis.scope.ignorePolicy}.`]
+      : []),
+    ...(analysis.sourceGeneration
+      ? [
+          `Source: ${analysis.sourceGeneration.verification}; ${String(analysis.sourceGeneration.filesUnavailable)} unavailable.`,
+        ]
+      : []),
+    ...(judge
+      ? [
+          `Semantic judge: ${judge.status}; ${String(judge.judgedCandidates)}/${String(judge.candidatesConsidered)} judged, ${String(judge.candidatesUnjudged)} unjudged; batches ${String(judge.batchesCompleted)}/${String(judge.batchesAttempted)} completed; classes ${JSON.stringify(judge.classificationCounts)}${judge.reason ? `; ${judge.reason}` : ""}.`,
+        ]
+      : []),
+    ...(details.operation
+      ? [`Operation: ${details.operation.state}; id=${details.operation.id}.`]
+      : []),
+    ...analysis.reasons.map((reason) => `[${reason}]`),
+    ...(details.redactionApplied ? ["[Display redaction applied.]"] : []),
+  ];
+}
+
 function compactMetadata(details: SiftLightDetails, analysis: AnalysisDetails): string[] {
+  if (analysis.kind === "concept" || analysis.kind === "hybrid")
+    return compactSemanticMetadata(details, analysis);
   return [
     ...(analysis.statistics ? formatStatistics(analysis.statistics) : []),
     analysis.counts ? `Counts: ${JSON.stringify(analysis.counts)}` : undefined,
@@ -46,7 +87,10 @@ function compactRows(analysis: AnalysisDetails): string[] {
       rows.push(JSON.stringify(item.path));
       previousPath = item.path;
     }
-    const label = analysis.kind === "outline" && analysis.modelOutput ? item.label : "metadata";
+    const label =
+      (analysis.kind === "outline" && analysis.modelOutput) || analysis.kind === "files"
+        ? item.label
+        : "metadata";
     const semanticJudge = item.details?.semanticJudge;
     const judgment =
       isRecord(semanticJudge) &&
@@ -108,6 +152,7 @@ export function compactMcpModelText(result: SiftLightResult): string {
   const inspect = compactInspectInstruction(analysis);
   const nextRequest = distinctNextRequest(result.details, analysis);
   const compact = [
+    ...(result.details.requestNotes ?? []).map((note) => `[Request note: ${note}]`),
     header,
     ...compactMetadata(result.details, analysis),
     ...compactRows(analysis),
@@ -115,6 +160,7 @@ export function compactMcpModelText(result: SiftLightResult): string {
     ...(nextRequest ? [`Next request: ${nextRequest}`] : []),
   ].join("\n");
   const standard = result.text.replace(" Structured output retains per-item evidence details.", "");
+  // Both projections carry request notes, so the byte comparison stays fair.
   if (analysis.semanticJudge) return compact;
   return Buffer.byteLength(compact) < Buffer.byteLength(standard) ? compact : standard;
 }

package/src/mcp-semantic-judge.ts

@@ -1,11 +1,12 @@
 import {
+  DEFAULT_SIFT_LIGHT_CONFIG,
   DEFAULT_SEMANTIC_JUDGE_CONFIG,
   readSiftLightConfigFile,
   SIFT_LIGHT_CONFIG_ENV,
 } from "./config-reader.js";
 import {
   createDisabledSemanticJudgeIntegration,
-  createSemanticJudgeIntegration,
+  createConfiguredSemanticJudgeIntegration,
   type SemanticJudgeIntegration,
 } from "./semantic-judge.js";
 
@@ -19,17 +20,27 @@ function configuredPath(environment: NodeJS.ProcessEnv): string | undefined {
  * The MCP process must receive the same config path explicitly in its own host
  * environment; an absent path remains an observable, local-only disabled state.
  */
-export async function createMcpSemanticJudgeIntegration(
+export async function createMcpSearchFeatures(
   environment: NodeJS.ProcessEnv = process.env,
-): Promise<SemanticJudgeIntegration> {
+): Promise<{ semanticJudge: SemanticJudgeIntegration; vectorSearchEnabled: boolean }> {
   const path = configuredPath(environment);
-  if (!path) return createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG);
+  if (!path)
+    return {
+      semanticJudge: createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG),
+      vectorSearchEnabled: DEFAULT_SIFT_LIGHT_CONFIG.vectorSearchEnabled === true,
+    };
 
   const config = await readSiftLightConfigFile(path, { missing: "error" });
-  return createSemanticJudgeIntegration(
-    config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG,
-    environment,
-  );
+  return {
+    semanticJudge: createConfiguredSemanticJudgeIntegration(config, environment),
+    vectorSearchEnabled: config.vectorSearchEnabled === true,
+  };
+}
+
+export async function createMcpSemanticJudgeIntegration(
+  environment: NodeJS.ProcessEnv = process.env,
+): Promise<SemanticJudgeIntegration> {
+  return (await createMcpSearchFeatures(environment)).semanticJudge;
 }
 
 export function mcpSemanticJudgeConfigSource(