pi-knowledge 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +147 -33
- package/dist/index.js +15 -4
- package/dist/src/embedding/provider.d.ts +14 -0
- package/dist/src/embedding/provider.js +90 -8
- package/dist/src/engine.d.ts +3 -1
- package/dist/src/engine.js +122 -147
- package/dist/src/indexer/chunker.d.ts +1 -1
- package/dist/src/indexer/chunker.js +2 -2
- package/dist/src/indexer/embedding-safe-chunks.d.ts +5 -0
- package/dist/src/indexer/embedding-safe-chunks.js +215 -0
- package/dist/src/indexer/html.d.ts +1 -0
- package/dist/src/indexer/html.js +49 -0
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +17 -1
- package/dist/src/model-worker.js +63 -15
- package/dist/src/search/contracts.d.ts +2 -0
- package/dist/src/search/contracts.js +2 -0
- package/dist/src/search/diversity.d.ts +16 -0
- package/dist/src/search/diversity.js +183 -0
- package/dist/src/search/tuning.d.ts +10 -0
- package/dist/src/search/tuning.js +20 -0
- package/dist/src/storage/sqlite.d.ts +1 -0
- package/dist/src/storage/sqlite.js +3 -0
- package/docs/configuration.md +3 -3
- package/package.json +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { MAX_SEARCH_RESULTS, MAX_SEARCH_WINDOW } from "./contracts.js";
|
|
1
2
|
import { MIN_HYBRID_SCORE } from "./ranking.js";
|
|
2
3
|
const PROFILE_DEFAULTS = {
|
|
3
4
|
balanced: {
|
|
@@ -152,6 +153,20 @@ function chooseRuntimeProfile(input) {
|
|
|
152
153
|
return "long_context";
|
|
153
154
|
return "balanced";
|
|
154
155
|
}
|
|
156
|
+
export function resolveSearchWindow(limit, offset, defaultLimit) {
|
|
157
|
+
const resolvedLimit = limit ?? defaultLimit;
|
|
158
|
+
if (!Number.isInteger(resolvedLimit) || resolvedLimit < 1 || resolvedLimit > MAX_SEARCH_RESULTS) {
|
|
159
|
+
throw new Error(`Search limit must be an integer from 1 to ${MAX_SEARCH_RESULTS}`);
|
|
160
|
+
}
|
|
161
|
+
const resolvedOffset = offset ?? 0;
|
|
162
|
+
if (!Number.isInteger(resolvedOffset) || resolvedOffset < 0) {
|
|
163
|
+
throw new Error("Search offset must be a non-negative integer");
|
|
164
|
+
}
|
|
165
|
+
if (resolvedOffset + resolvedLimit > MAX_SEARCH_WINDOW) {
|
|
166
|
+
throw new Error(`Search offset + limit must not exceed ${MAX_SEARCH_WINDOW}; narrow the query or use a more specific knowledge base`);
|
|
167
|
+
}
|
|
168
|
+
return { limit: resolvedLimit, offset: resolvedOffset };
|
|
169
|
+
}
|
|
155
170
|
export function resolveSearchTuning(input) {
|
|
156
171
|
const requestedProfile = input.profile ?? parseProfile(process.env.PI_KNOWLEDGE_SEARCH_PROFILE) ?? "auto";
|
|
157
172
|
const selectedProfile = requestedProfile === "auto" ? chooseRuntimeProfile(input) : requestedProfile;
|
|
@@ -179,6 +194,11 @@ export function summarizeSearchTuning(tuning, limit, candidateLimit) {
|
|
|
179
194
|
snippet_max_length: tuning.snippetMaxLength,
|
|
180
195
|
min_hybrid_score: tuning.minHybridScore,
|
|
181
196
|
candidate_limit: candidateLimit,
|
|
197
|
+
retrieval_limit_per_kb: candidateLimit,
|
|
198
|
+
retrieved_candidates: 0,
|
|
199
|
+
filtered_candidates: 0,
|
|
200
|
+
diversity_candidates: 0,
|
|
201
|
+
candidate_pool_truncated: false,
|
|
182
202
|
candidate_min: tuning.candidateMin,
|
|
183
203
|
candidate_multiplier: tuning.candidateMultiplier,
|
|
184
204
|
adaptive_context_lines: tuning.adaptiveContextLines,
|
|
@@ -87,6 +87,7 @@ export declare function listKBs(db: Database.Database): KnowledgeBase[];
|
|
|
87
87
|
export declare function deleteKB(db: Database.Database, id: string): void;
|
|
88
88
|
export declare function updateKBStatus(db: Database.Database, id: string, status: KnowledgeBase["status"]): void;
|
|
89
89
|
export declare function updateKBCounts(db: Database.Database, id: string, chunkCount: number, fileCount: number): void;
|
|
90
|
+
export declare function updateKBSourceOptions(db: Database.Database, id: string, sourceOptions: string): void;
|
|
90
91
|
export declare function updateKBEmbeddingMetadata(db: Database.Database, id: string, embeddingModel: string, embeddingSignature: string | null, embeddingDimension: number | null): void;
|
|
91
92
|
export declare function startIndexingJob(db: Database.Database, kbId: string, operation: IndexingJob["operation"], message: string): void;
|
|
92
93
|
export declare function updateIndexingJob(db: Database.Database, kbId: string, progress: {
|
|
@@ -355,6 +355,9 @@ export function updateKBStatus(db, id, status) {
|
|
|
355
355
|
export function updateKBCounts(db, id, chunkCount, fileCount) {
|
|
356
356
|
db.prepare("UPDATE knowledge_bases SET chunk_count = ?, file_count = ?, updated_at = ? WHERE id = ?").run(chunkCount, fileCount, Date.now(), id);
|
|
357
357
|
}
|
|
358
|
+
export function updateKBSourceOptions(db, id, sourceOptions) {
|
|
359
|
+
db.prepare("UPDATE knowledge_bases SET source_options = ?, updated_at = ? WHERE id = ?").run(sourceOptions, Date.now(), id);
|
|
360
|
+
}
|
|
358
361
|
export function updateKBEmbeddingMetadata(db, id, embeddingModel, embeddingSignature, embeddingDimension) {
|
|
359
362
|
db.prepare(`UPDATE knowledge_bases
|
|
360
363
|
SET embedding_model = ?, embedding_signature = ?, embedding_dimension = ?, updated_at = ?
|
package/docs/configuration.md
CHANGED
|
@@ -33,7 +33,7 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
|
|
|
33
33
|
| `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
|
|
34
34
|
| `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
|
|
35
35
|
|
|
36
|
-
Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, query/document prefixes, non-default prefix strategy, pooling, normalization, API input cap, and a hash of the API base URL when applicable. `knowledge_search` skips vector retrieval for KBs whose stored signature or dimension is missing or incompatible with the current query embedding and uses BM25-only fallback for hybrid searches. Run `knowledge_update` to rebuild vectors after changing embedding provider/model/max-character/prefix settings or after upgrading
|
|
36
|
+
Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, query/document prefixes, non-default prefix strategy, pooling, normalization policy, API input cap, and a hash of the API base URL when applicable. OpenAI-compatible response vectors are validated and L2-normalized by the client before use or persistence; the `client-l2-v1` policy is part of API signatures. `knowledge_search` skips vector retrieval for KBs whose stored signature or dimension is missing or incompatible with the current query embedding and uses BM25-only fallback for hybrid searches. Run `knowledge_update` to rebuild vectors after changing embedding provider/model/max-character/prefix settings or after upgrading an API-backed KB created before client-side normalization was signed.
|
|
37
37
|
|
|
38
38
|
## Reranker
|
|
39
39
|
|
|
@@ -83,9 +83,9 @@ API embedding failures intentionally surface by default. Silent fallback can hid
|
|
|
83
83
|
| `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES` | profile default | Override how many filtered candidates `deep` mode sends to the reranker. Clamped to 5-100. |
|
|
84
84
|
| `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` | profile default | Override deep rerank top-k multiplier relative to the effective result limit. Clamped to 1-10. |
|
|
85
85
|
|
|
86
|
-
Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, candidate pool
|
|
86
|
+
Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, global candidate limit, per-KB retrieval limit, retrieved/filtered/diversity candidate counts, global-pool truncation, chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
|
|
87
87
|
|
|
88
|
-
`knowledge_search` supports `file_type`, `path_pattern`, `profile`, and
|
|
88
|
+
`knowledge_search` supports `file_type`, substring `path_pattern`, `profile`, integer `limit` from 1 to 50, and non-negative integer `offset`. `offset + limit` cannot exceed the 800-candidate global search window. Each KB retrieves up to the profile-derived candidate limit, then merged results are ranked, filtered, and globally capped before deep reranking, adaptive expansion, or diversity. Search profiles tune result count, snippets, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth at runtime. `auto` chooses from query shape, mode, and KB source type; explicit tool parameters win over env overrides, env overrides win over profile defaults, and profile defaults win over built-in compatibility values. `hybrid` mode remains lexical-anchored to prevent low-evidence semantic false positives; use `semantic` for pure vector search, or `semantic_hybrid` when weak lexical overlap should not block vector-backed weighted fusion. `adaptive` uses bounded same-file context expansion; when AST metadata is present it can prefer same-parent/sibling chunks without claiming caller/callee graph semantics.
|
|
89
89
|
|
|
90
90
|
`knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
|
|
91
91
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|