skilld 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -23
- package/dist/_chunks/config.mjs +8 -2
- package/dist/_chunks/config.mjs.map +1 -1
- package/dist/_chunks/llm.mjs +710 -204
- package/dist/_chunks/llm.mjs.map +1 -1
- package/dist/_chunks/pool.mjs +115 -0
- package/dist/_chunks/pool.mjs.map +1 -0
- package/dist/_chunks/releases.mjs +689 -179
- package/dist/_chunks/releases.mjs.map +1 -1
- package/dist/_chunks/storage.mjs +311 -19
- package/dist/_chunks/storage.mjs.map +1 -1
- package/dist/_chunks/sync-parallel.mjs +134 -378
- package/dist/_chunks/sync-parallel.mjs.map +1 -1
- package/dist/_chunks/types.d.mts +9 -6
- package/dist/_chunks/types.d.mts.map +1 -1
- package/dist/_chunks/utils.d.mts +137 -68
- package/dist/_chunks/utils.d.mts.map +1 -1
- package/dist/_chunks/version.d.mts +43 -6
- package/dist/_chunks/version.d.mts.map +1 -1
- package/dist/agent/index.d.mts +58 -15
- package/dist/agent/index.d.mts.map +1 -1
- package/dist/agent/index.mjs +4 -2
- package/dist/cache/index.d.mts +2 -2
- package/dist/cache/index.mjs +2 -2
- package/dist/cli.mjs +2170 -1436
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +4 -3
- package/dist/index.mjs +2 -2
- package/dist/retriv/index.d.mts +16 -2
- package/dist/retriv/index.d.mts.map +1 -1
- package/dist/retriv/index.mjs +44 -15
- package/dist/retriv/index.mjs.map +1 -1
- package/dist/retriv/worker.d.mts +33 -0
- package/dist/retriv/worker.d.mts.map +1 -0
- package/dist/retriv/worker.mjs +47 -0
- package/dist/retriv/worker.mjs.map +1 -0
- package/dist/sources/index.d.mts +2 -2
- package/dist/sources/index.mjs +2 -2
- package/dist/types.d.mts +5 -3
- package/package.json +11 -7
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { A as CACHE_DIR, D as CacheConfig, E as writeToCache, M as SEARCH_DB, O as CachedDoc, S as readCachedDocs, a as clearAllCache, b as listCached, d as isCached, g as linkReferences, j as REFERENCES_DIR, k as CachedPackage, n as getCacheKey, o as clearCache, r as getVersionKey, s as ensureCacheDir, t as getCacheDir } from "./_chunks/version.mjs";
|
|
2
|
+
import { c as SearchResult, l as SearchSnippet, n as Document, o as SearchFilter, r as IndexConfig, s as SearchOptions } from "./_chunks/types.mjs";
|
|
3
3
|
import { createIndex, search, searchSnippets } from "./retriv/index.mjs";
|
|
4
|
-
import { A as
|
|
4
|
+
import { $ as NpmPackageInfo, A as downloadLlmsDocs, C as readLocalDependencies, D as resolvePackageDocs, F as parseMarkdownLinks, K as fetchReadmeContent, M as fetchLlmsTxt, P as normalizeLlmsLinks, Q as LocalDependency, X as LlmsContent, Y as FetchedDoc, Z as LlmsLink, nt as ResolvedPackage, v as fetchNpmPackage } from "./_chunks/utils.mjs";
|
|
5
|
+
import "./sources/index.mjs";
|
|
5
6
|
export { CACHE_DIR, type CacheConfig, type CachedDoc, type CachedPackage, type Document, type FetchedDoc, type IndexConfig, type LlmsContent, type LlmsLink, type LocalDependency, type NpmPackageInfo, REFERENCES_DIR, type ResolvedPackage, SEARCH_DB, type SearchFilter, type SearchOptions, type SearchResult, type SearchSnippet, clearAllCache, clearCache, createIndex, downloadLlmsDocs, ensureCacheDir, fetchLlmsTxt, fetchNpmPackage, fetchReadmeContent, getCacheDir, getCacheKey, getVersionKey, isCached, linkReferences, listCached, normalizeLlmsLinks, parseMarkdownLinks, readCachedDocs, readLocalDependencies, resolvePackageDocs, search, searchSnippets, writeToCache };
|
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { a as getCacheDir, n as REFERENCES_DIR, o as getCacheKey, r as SEARCH_DB, s as getVersionKey, t as CACHE_DIR } from "./_chunks/config.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { S as writeToCache, f as linkReferences, g as listCached, n as clearCache, r as ensureCacheDir, s as isCached, t as clearAllCache, v as readCachedDocs } from "./_chunks/storage.mjs";
|
|
3
3
|
import "./cache/index.mjs";
|
|
4
4
|
import { createIndex, search, searchSnippets } from "./retriv/index.mjs";
|
|
5
|
-
import {
|
|
5
|
+
import { E as fetchReadmeContent, b as normalizeLlmsLinks, g as downloadLlmsDocs, i as fetchNpmPackage, l as readLocalDependencies, p as resolvePackageDocs, v as fetchLlmsTxt, x as parseMarkdownLinks } from "./_chunks/releases.mjs";
|
|
6
6
|
import "./sources/index.mjs";
|
|
7
7
|
export { CACHE_DIR, REFERENCES_DIR, SEARCH_DB, clearAllCache, clearCache, createIndex, downloadLlmsDocs, ensureCacheDir, fetchLlmsTxt, fetchNpmPackage, fetchReadmeContent, getCacheDir, getCacheKey, getVersionKey, isCached, linkReferences, listCached, normalizeLlmsLinks, parseMarkdownLinks, readCachedDocs, readLocalDependencies, resolvePackageDocs, search, searchSnippets, writeToCache };
|
package/dist/retriv/index.d.mts
CHANGED
|
@@ -1,12 +1,26 @@
|
|
|
1
|
-
import { a as
|
|
1
|
+
import { a as IndexProgress, c as SearchResult, i as IndexPhase, l as SearchSnippet, n as Document, o as SearchFilter, r as IndexConfig, s as SearchOptions, t as ChunkEntity } from "../_chunks/types.mjs";
|
|
2
|
+
import { createRetriv } from "retriv";
|
|
2
3
|
|
|
3
4
|
//#region src/retriv/index.d.ts
|
|
5
|
+
type RetrivInstance = Awaited<ReturnType<typeof createRetriv>>;
|
|
6
|
+
/**
|
|
7
|
+
* Index documents in-process (no worker thread).
|
|
8
|
+
* Preferred for tests and environments where worker_threads is unreliable.
|
|
9
|
+
*/
|
|
10
|
+
declare function createIndexDirect(documents: Document[], config: IndexConfig): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Index documents in a background worker thread.
|
|
13
|
+
* Falls back to direct indexing if worker fails to spawn.
|
|
14
|
+
*/
|
|
4
15
|
declare function createIndex(documents: Document[], config: IndexConfig): Promise<void>;
|
|
5
16
|
declare function search(query: string, config: IndexConfig, options?: SearchOptions): Promise<SearchResult[]>;
|
|
6
17
|
/**
|
|
7
18
|
* Search and return formatted snippets
|
|
8
19
|
*/
|
|
9
20
|
declare function searchSnippets(query: string, config: IndexConfig, options?: SearchOptions): Promise<SearchSnippet[]>;
|
|
21
|
+
declare function openPool(dbPaths: string[]): Promise<Map<string, RetrivInstance>>;
|
|
22
|
+
declare function searchPooled(query: string, pool: Map<string, RetrivInstance>, options?: SearchOptions): Promise<SearchSnippet[]>;
|
|
23
|
+
declare function closePool(pool: Map<string, RetrivInstance>): Promise<void>;
|
|
10
24
|
//#endregion
|
|
11
|
-
export { type ChunkEntity, type Document, type IndexConfig, type SearchFilter, type SearchOptions, type SearchResult, type SearchSnippet, createIndex, search, searchSnippets };
|
|
25
|
+
export { type ChunkEntity, type Document, type IndexConfig, type IndexPhase, type IndexProgress, type SearchFilter, type SearchOptions, type SearchResult, type SearchSnippet, closePool, createIndex, createIndexDirect, openPool, search, searchPooled, searchSnippets };
|
|
12
26
|
//# sourceMappingURL=index.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/retriv/index.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/retriv/index.ts"],"mappings":";;;;KAQK,cAAA,GAAiB,OAAA,CAAQ,UAAA,QAAkB,YAAA;AAFuF;;;;AAAA,iBAkBjH,iBAAA,CACpB,SAAA,EAAW,QAAA,IACX,MAAA,EAAQ,WAAA,GACP,OAAA;;;;;iBAUmB,WAAA,CACpB,SAAA,EAAW,QAAA,IACX,MAAA,EAAQ,WAAA,GACP,OAAA;AAAA,iBAMmB,MAAA,CACpB,KAAA,UACA,MAAA,EAAQ,WAAA,EACR,OAAA,GAAS,aAAA,GACR,OAAA,CAAQ,YAAA;;;AA1BX;iBAuDsB,cAAA,CACpB,KAAA,UACA,MAAA,EAAQ,WAAA,EACR,OAAA,GAAS,aAAA,GACR,OAAA,CAAQ,aAAA;AAAA,iBA2BW,QAAA,CAAS,OAAA,aAAoB,OAAA,CAAQ,GAAA,SAAY,cAAA;AAAA,iBASjD,YAAA,CACpB,KAAA,UACA,IAAA,EAAM,GAAA,SAAY,cAAA,GAClB,OAAA,GAAS,aAAA,GACR,OAAA,CAAQ,aAAA;AAAA,iBAqBW,SAAA,CAAU,IAAA,EAAM,GAAA,SAAY,cAAA,IAAkB,OAAA"}
|
package/dist/retriv/index.mjs
CHANGED
|
@@ -11,22 +11,15 @@ function getDb(config) {
|
|
|
11
11
|
chunking: autoChunker()
|
|
12
12
|
});
|
|
13
13
|
}
|
|
14
|
-
async function
|
|
14
|
+
async function createIndexDirect(documents, config) {
|
|
15
15
|
const db = await getDb(config);
|
|
16
|
-
|
|
17
|
-
let indexed = 0;
|
|
18
|
-
for (let i = 0; i < documents.length; i += BATCH_SIZE) {
|
|
19
|
-
const batch = documents.slice(i, i + BATCH_SIZE);
|
|
20
|
-
await db.index(batch);
|
|
21
|
-
indexed += batch.length;
|
|
22
|
-
const last = batch[batch.length - 1];
|
|
23
|
-
config.onProgress?.(indexed, documents.length, {
|
|
24
|
-
id: last.id,
|
|
25
|
-
type: last.metadata?.type
|
|
26
|
-
});
|
|
27
|
-
}
|
|
16
|
+
await db.index(documents, { onProgress: config.onProgress });
|
|
28
17
|
await db.close?.();
|
|
29
18
|
}
|
|
19
|
+
async function createIndex(documents, config) {
|
|
20
|
+
const { createIndexInWorker } = await import("../_chunks/pool.mjs");
|
|
21
|
+
return createIndexInWorker(documents, config);
|
|
22
|
+
}
|
|
30
23
|
async function search(query, config, options = {}) {
|
|
31
24
|
const { limit = 10, filter } = options;
|
|
32
25
|
const db = await getDb(config);
|
|
@@ -54,7 +47,10 @@ function stripFrontmatter(content) {
|
|
|
54
47
|
return match ? content.slice(match[0].length).trim() : content;
|
|
55
48
|
}
|
|
56
49
|
async function searchSnippets(query, config, options = {}) {
|
|
57
|
-
return (await search(query, config, options))
|
|
50
|
+
return toSnippets(await search(query, config, options));
|
|
51
|
+
}
|
|
52
|
+
function toSnippets(results) {
|
|
53
|
+
return results.map((r) => {
|
|
58
54
|
const content = stripFrontmatter(r.content);
|
|
59
55
|
const source = r.metadata.source || r.id;
|
|
60
56
|
const lines = content.split("\n").length;
|
|
@@ -71,6 +67,39 @@ async function searchSnippets(query, config, options = {}) {
|
|
|
71
67
|
};
|
|
72
68
|
});
|
|
73
69
|
}
|
|
74
|
-
|
|
70
|
+
async function openPool(dbPaths) {
|
|
71
|
+
const pool = /* @__PURE__ */ new Map();
|
|
72
|
+
await Promise.all(dbPaths.map(async (dbPath) => {
|
|
73
|
+
const db = await getDb({ dbPath });
|
|
74
|
+
pool.set(dbPath, db);
|
|
75
|
+
}));
|
|
76
|
+
return pool;
|
|
77
|
+
}
|
|
78
|
+
async function searchPooled(query, pool, options = {}) {
|
|
79
|
+
const { limit = 10, filter } = options;
|
|
80
|
+
return toSnippets((await Promise.all([...pool.values()].map(async (db) => {
|
|
81
|
+
return (await db.search(query, {
|
|
82
|
+
limit,
|
|
83
|
+
filter,
|
|
84
|
+
returnContent: true,
|
|
85
|
+
returnMetadata: true,
|
|
86
|
+
returnMeta: true
|
|
87
|
+
})).map((r) => ({
|
|
88
|
+
id: r.id,
|
|
89
|
+
content: r.content ?? "",
|
|
90
|
+
score: r.score,
|
|
91
|
+
metadata: r.metadata ?? {},
|
|
92
|
+
highlights: r._meta?.highlights ?? [],
|
|
93
|
+
lineRange: r._chunk?.lineRange,
|
|
94
|
+
entities: r._chunk?.entities,
|
|
95
|
+
scope: r._chunk?.scope
|
|
96
|
+
}));
|
|
97
|
+
}))).flat().sort((a, b) => b.score - a.score).slice(0, limit));
|
|
98
|
+
}
|
|
99
|
+
async function closePool(pool) {
|
|
100
|
+
await Promise.all([...pool.values()].map((db) => db.close?.()));
|
|
101
|
+
pool.clear();
|
|
102
|
+
}
|
|
103
|
+
export { closePool, createIndex, createIndexDirect, openPool, search, searchPooled, searchSnippets };
|
|
75
104
|
|
|
76
105
|
//# sourceMappingURL=index.mjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","names":[],"sources":["../../src/retriv/index.ts"],"sourcesContent":["import type { ChunkEntity, Document, IndexConfig, SearchFilter, SearchOptions, SearchResult, SearchSnippet } from './types'\nimport { createRetriv } from 'retriv'\nimport { autoChunker } from 'retriv/chunkers/auto'\nimport sqlite from 'retriv/db/sqlite'\nimport { transformersJs } from 'retriv/embeddings/transformers-js'\n\nexport type { ChunkEntity, Document, IndexConfig, SearchFilter, SearchOptions, SearchResult, SearchSnippet }\n\nfunction getDb(config: IndexConfig) {\n return createRetriv({\n driver: sqlite({\n path: config.dbPath,\n embeddings: transformersJs(),\n }),\n chunking: autoChunker(),\n })\n}\n\nexport async function
|
|
1
|
+
{"version":3,"file":"index.mjs","names":[],"sources":["../../src/retriv/index.ts"],"sourcesContent":["import type { ChunkEntity, Document, IndexConfig, IndexPhase, IndexProgress, SearchFilter, SearchOptions, SearchResult, SearchSnippet } from './types'\nimport { createRetriv } from 'retriv'\nimport { autoChunker } from 'retriv/chunkers/auto'\nimport sqlite from 'retriv/db/sqlite'\nimport { transformersJs } from 'retriv/embeddings/transformers-js'\n\nexport type { ChunkEntity, Document, IndexConfig, IndexPhase, IndexProgress, SearchFilter, SearchOptions, SearchResult, SearchSnippet }\n\ntype RetrivInstance = Awaited<ReturnType<typeof createRetriv>>\n\nfunction getDb(config: IndexConfig) {\n return createRetriv({\n driver: sqlite({\n path: config.dbPath,\n embeddings: transformersJs(),\n }),\n chunking: autoChunker(),\n })\n}\n\n/**\n * Index documents in-process (no worker thread).\n * Preferred for tests and environments where worker_threads is unreliable.\n */\nexport async function createIndexDirect(\n documents: Document[],\n config: IndexConfig,\n): Promise<void> {\n const db = await getDb(config)\n await db.index(documents, { onProgress: config.onProgress })\n await db.close?.()\n}\n\n/**\n * Index documents in a background worker thread.\n * Falls back to direct indexing if worker fails to spawn.\n */\nexport async function createIndex(\n documents: Document[],\n config: IndexConfig,\n): Promise<void> {\n // Dynamic import justified: search/searchSnippets shouldn't pull in worker_threads\n const { createIndexInWorker } = await import('./pool')\n return createIndexInWorker(documents, config)\n}\n\nexport async function search(\n query: string,\n config: IndexConfig,\n options: SearchOptions = {},\n): Promise<SearchResult[]> {\n const { limit = 10, filter } = options\n const db = await getDb(config)\n const results = await db.search(query, { limit, filter, returnContent: true, returnMetadata: true, returnMeta: true })\n await db.close?.()\n\n return results.map(r => ({\n id: r.id,\n content: r.content ?? '',\n score: r.score,\n metadata: r.metadata ?? {},\n highlights: r._meta?.highlights ?? [],\n lineRange: r._chunk?.lineRange,\n entities: r._chunk?.entities,\n scope: r._chunk?.scope,\n }))\n}\n\n/**\n * Strip YAML frontmatter from markdown content\n */\nfunction stripFrontmatter(content: string): string {\n const match = content.match(/^---\\r?\\n[\\s\\S]*?\\r?\\n---\\r?\\n/)\n return match ? content.slice(match[0].length).trim() : content\n}\n\n/**\n * Search and return formatted snippets\n */\nexport async function searchSnippets(\n query: string,\n config: IndexConfig,\n options: SearchOptions = {},\n): Promise<SearchSnippet[]> {\n const results = await search(query, config, options)\n return toSnippets(results)\n}\n\nfunction toSnippets(results: SearchResult[]): SearchSnippet[] {\n return results.map((r) => {\n const content = stripFrontmatter(r.content)\n const source = r.metadata.source || r.id\n const lines = content.split('\\n').length\n\n return {\n package: r.metadata.package || 'unknown',\n source,\n lineStart: r.lineRange?.[0] ?? 1,\n lineEnd: r.lineRange?.[1] ?? lines,\n content,\n score: r.score,\n highlights: r.highlights,\n entities: r.entities,\n scope: r.scope,\n }\n })\n}\n\n// ── Pooled DB access for interactive search ──\n\nexport async function openPool(dbPaths: string[]): Promise<Map<string, RetrivInstance>> {\n const pool = new Map<string, RetrivInstance>()\n await Promise.all(dbPaths.map(async (dbPath) => {\n const db = await getDb({ dbPath })\n pool.set(dbPath, db)\n }))\n return pool\n}\n\nexport async function searchPooled(\n query: string,\n pool: Map<string, RetrivInstance>,\n options: SearchOptions = {},\n): Promise<SearchSnippet[]> {\n const { limit = 10, filter } = options\n const allResults = await Promise.all(\n [...pool.values()].map(async (db) => {\n const results = await db.search(query, { limit, filter, returnContent: true, returnMetadata: true, returnMeta: true })\n return results.map(r => ({\n id: r.id,\n content: r.content ?? '',\n score: r.score,\n metadata: r.metadata ?? {},\n highlights: r._meta?.highlights ?? [],\n lineRange: r._chunk?.lineRange as [number, number] | undefined,\n entities: r._chunk?.entities,\n scope: r._chunk?.scope,\n }))\n }),\n )\n const merged = allResults.flat().sort((a, b) => b.score - a.score).slice(0, limit)\n return toSnippets(merged)\n}\n\nexport async function closePool(pool: Map<string, RetrivInstance>): Promise<void> {\n await Promise.all([...pool.values()].map(db => db.close?.()))\n pool.clear()\n}\n"],"mappings":";;;;AAUA,SAAS,MAAM,QAAqB;AAClC,QAAO,aAAa;EAClB,QAAQ,OAAO;GACb,MAAM,OAAO;GACb,YAAY,gBAAA;GACb,CAAC;EACF,UAAU,aAAA;EACX,CAAC;;AAOJ,eAAsB,kBACpB,WACA,QACe;CACf,MAAM,KAAK,MAAM,MAAM,OAAO;AAC9B,OAAM,GAAG,MAAM,WAAW,EAAE,YAAY,OAAO,YAAY,CAAC;AAC5D,OAAM,GAAG,SAAS;;AAOpB,eAAsB,YACpB,WACA,QACe;CAEf,MAAM,EAAE,wBAAwB,MAAM,OAAO;AAC7C,QAAO,oBAAoB,WAAW,OAAO;;AAG/C,eAAsB,OACpB,OACA,QACA,UAAyB,EAAE,EACF;CACzB,MAAM,EAAE,QAAQ,IAAI,WAAW;CAC/B,MAAM,KAAK,MAAM,MAAM,OAAO;CAC9B,MAAM,UAAU,MAAM,GAAG,OAAO,OAAO;EAAE;EAAO;EAAQ,eAAe;EAAM,gBAAgB;EAAM,YAAY;EAAM,CAAC;AACtH,OAAM,GAAG,SAAS;AAElB,QAAO,QAAQ,KAAI,OAAM;EACvB,IAAI,EAAE;EACN,SAAS,EAAE,WAAW;EACtB,OAAO,EAAE;EACT,UAAU,EAAE,YAAY,EAAE;EAC1B,YAAY,EAAE,OAAO,cAAc,EAAE;EACrC,WAAW,EAAE,QAAQ;EACrB,UAAU,EAAE,QAAQ;EACpB,OAAO,EAAE,QAAQ;EAClB,EAAE;;AAML,SAAS,iBAAiB,SAAyB;CACjD,MAAM,QAAQ,QAAQ,MAAM,iCAAiC;AAC7D,QAAO,QAAQ,QAAQ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG;;AAMzD,eAAsB,eACpB,OACA,QACA,UAAyB,EAAE,EACD;AAE1B,QAAO,WADS,MAAM,OAAO,OAAO,QAAQ,QAAQ,CAC1B;;AAG5B,SAAS,WAAW,SAA0C;AAC5D,QAAO,QAAQ,KAAK,MAAM;EACxB,MAAM,UAAU,iBAAiB,EAAE,QAAQ;EAC3C,MAAM,SAAS,EAAE,SAAS,UAAU,EAAE;EACtC,MAAM,QAAQ,QAAQ,MAAM,KAAK,CAAC;AAElC,SAAO;GACL,SAAS,EAAE,SAAS,WAAW;GAC/B;GACA,WAAW,EAAE,YAAY,MAAM;GAC/B,SAAS,EAAE,YAAY,MAAM;GAC7B;GACA,OAAO,EAAE;GACT,YAAY,EAAE;GACd,UAAU,EAAE;GACZ,OAAO,EAAE;GACV;GACD;;AAKJ,eAAsB,SAAS,SAAyD;CACtF,MAAM,uBAAO,IAAI,KAA6B;AAC9C,OAAM,QAAQ,IAAI,QAAQ,IAAI,OAAO,WAAW;EAC9C,MAAM,KAAK,MAAM,MAAM,EAAE,QAAQ,CAAC;AAClC,OAAK,IAAI,QAAQ,GAAG;GACpB,CAAC;AACH,QAAO;;AAGT,eAAsB,aACpB,OACA,MACA,UAAyB,EAAE,EACD;CAC1B,MAAM,EAAE,QAAQ,IAAI,WAAW;AAiB/B,QAAO,YAhBY,MAAM,QAAQ,IAC/B,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,IAAI,OAAO,OAAO;AAEnC,UADgB,MAAM,GAAG,OAAO,OAAO;GAAE;GAAO;GAAQ,eAAe;GAAM,gBAAgB;GAAM,YAAY;GAAM,CAAC,EACvG,KAAI,OAAM;GACvB,IAAI,EAAE;GACN,SAAS,EAAE,WAAW;GACtB,OAAO,EAAE;GACT,UAAU,EAAE,YAAY,EAAE;GAC1B,YAAY,EAAE,OAAO,cAAc,EAAE;GACrC,WAAW,EAAE,QAAQ;GACrB,UAAU,EAAE,QAAQ;GACpB,OAAO,EAAE,QAAQ;GAClB,EAAE;GACH,CACH,EACyB,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CACzD;;AAG3B,eAAsB,UAAU,MAAkD;AAChF,OAAM,QAAQ,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,KAAI,OAAM,GAAG,SAAS,CAAC,CAAC;AAC7D,MAAK,OAAO"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { n as Document } from "../_chunks/types.mjs";
|
|
2
|
+
|
|
3
|
+
//#region src/retriv/worker.d.ts
|
|
4
|
+
interface WorkerIndexMessage {
|
|
5
|
+
type: 'index';
|
|
6
|
+
id: number;
|
|
7
|
+
documents: Document[];
|
|
8
|
+
dbPath: string;
|
|
9
|
+
}
|
|
10
|
+
interface WorkerShutdownMessage {
|
|
11
|
+
type: 'shutdown';
|
|
12
|
+
}
|
|
13
|
+
type WorkerMessage = WorkerIndexMessage | WorkerShutdownMessage;
|
|
14
|
+
interface WorkerProgressResponse {
|
|
15
|
+
type: 'progress';
|
|
16
|
+
id: number;
|
|
17
|
+
phase: string;
|
|
18
|
+
current: number;
|
|
19
|
+
total: number;
|
|
20
|
+
}
|
|
21
|
+
interface WorkerDoneResponse {
|
|
22
|
+
type: 'done';
|
|
23
|
+
id: number;
|
|
24
|
+
}
|
|
25
|
+
interface WorkerErrorResponse {
|
|
26
|
+
type: 'error';
|
|
27
|
+
id: number;
|
|
28
|
+
message: string;
|
|
29
|
+
}
|
|
30
|
+
type WorkerResponse = WorkerProgressResponse | WorkerDoneResponse | WorkerErrorResponse;
|
|
31
|
+
//#endregion
|
|
32
|
+
export { WorkerDoneResponse, WorkerErrorResponse, WorkerIndexMessage, WorkerMessage, WorkerProgressResponse, WorkerResponse, WorkerShutdownMessage };
|
|
33
|
+
//# sourceMappingURL=worker.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.mts","names":[],"sources":["../../src/retriv/worker.ts"],"mappings":";;;UAOiB,kBAAA;EACf,IAAA;EACA,EAAA;EACA,SAAA,EAAW,QAAA;EACX,MAAA;AAAA;AAAA,UAGe,qBAAA;EACf,IAAA;AAAA;AAAA,KAGU,aAAA,GAAgB,kBAAA,GAAqB,qBAAA;AAAA,UAEhC,sBAAA;EACf,IAAA;EACA,EAAA;EACA,KAAA;EACA,OAAA;EACA,KAAA;AAAA;AAAA,UAGe,kBAAA;EACf,IAAA;EACA,EAAA;AAAA;AAAA,UAGe,mBAAA;EACf,IAAA;EACA,EAAA;EACA,OAAA;AAAA;AAAA,KAGU,cAAA,GAAiB,sBAAA,GAAyB,kBAAA,GAAqB,mBAAA"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { createRetriv } from "retriv";
|
|
2
|
+
import { autoChunker } from "retriv/chunkers/auto";
|
|
3
|
+
import sqlite from "retriv/db/sqlite";
|
|
4
|
+
import { transformersJs } from "retriv/embeddings/transformers-js";
|
|
5
|
+
import { parentPort } from "node:worker_threads";
|
|
6
|
+
if (parentPort) parentPort.on("message", async (msg) => {
|
|
7
|
+
if (msg.type === "shutdown") process.exit(0);
|
|
8
|
+
if (msg.type === "index") {
|
|
9
|
+
const { id, documents, dbPath } = msg;
|
|
10
|
+
try {
|
|
11
|
+
const config = {
|
|
12
|
+
dbPath,
|
|
13
|
+
onProgress: ({ phase, current, total }) => {
|
|
14
|
+
parentPort.postMessage({
|
|
15
|
+
type: "progress",
|
|
16
|
+
id,
|
|
17
|
+
phase,
|
|
18
|
+
current,
|
|
19
|
+
total
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
const db = await createRetriv({
|
|
24
|
+
driver: sqlite({
|
|
25
|
+
path: config.dbPath,
|
|
26
|
+
embeddings: transformersJs()
|
|
27
|
+
}),
|
|
28
|
+
chunking: autoChunker()
|
|
29
|
+
});
|
|
30
|
+
await db.index(documents, { onProgress: config.onProgress });
|
|
31
|
+
await db.close?.();
|
|
32
|
+
parentPort.postMessage({
|
|
33
|
+
type: "done",
|
|
34
|
+
id
|
|
35
|
+
});
|
|
36
|
+
} catch (err) {
|
|
37
|
+
parentPort.postMessage({
|
|
38
|
+
type: "error",
|
|
39
|
+
id,
|
|
40
|
+
message: err instanceof Error ? err.message : String(err)
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
export {};
|
|
46
|
+
|
|
47
|
+
//# sourceMappingURL=worker.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.mjs","names":[],"sources":["../../src/retriv/worker.ts"],"sourcesContent":["import type { IndexConfig, Document as RetrivDocument } from './types'\nimport { parentPort } from 'node:worker_threads'\nimport { createRetriv } from 'retriv'\nimport { autoChunker } from 'retriv/chunkers/auto'\nimport sqlite from 'retriv/db/sqlite'\nimport { transformersJs } from 'retriv/embeddings/transformers-js'\n\nexport interface WorkerIndexMessage {\n type: 'index'\n id: number\n documents: RetrivDocument[]\n dbPath: string\n}\n\nexport interface WorkerShutdownMessage {\n type: 'shutdown'\n}\n\nexport type WorkerMessage = WorkerIndexMessage | WorkerShutdownMessage\n\nexport interface WorkerProgressResponse {\n type: 'progress'\n id: number\n phase: string\n current: number\n total: number\n}\n\nexport interface WorkerDoneResponse {\n type: 'done'\n id: number\n}\n\nexport interface WorkerErrorResponse {\n type: 'error'\n id: number\n message: string\n}\n\nexport type WorkerResponse = WorkerProgressResponse | WorkerDoneResponse | WorkerErrorResponse\n\nif (parentPort) {\n parentPort.on('message', async (msg: WorkerMessage) => {\n if (msg.type === 'shutdown') {\n process.exit(0)\n }\n\n if (msg.type === 'index') {\n const { id, documents, dbPath } = msg\n\n try {\n const config: IndexConfig = {\n dbPath,\n onProgress: ({ phase, current, total }) => {\n parentPort!.postMessage({ type: 'progress', id, phase, current, total } satisfies WorkerProgressResponse)\n },\n }\n\n const db = await createRetriv({\n driver: sqlite({\n path: config.dbPath,\n embeddings: transformersJs(),\n }),\n chunking: autoChunker(),\n })\n\n await db.index(documents, { onProgress: config.onProgress })\n await db.close?.()\n\n parentPort!.postMessage({ type: 'done', id } satisfies WorkerDoneResponse)\n }\n catch (err) {\n parentPort!.postMessage({\n type: 'error',\n id,\n message: err instanceof Error ? err.message : String(err),\n } satisfies WorkerErrorResponse)\n }\n }\n })\n}\n"],"mappings":";;;;;AAyCA,IAAI,WACF,YAAW,GAAG,WAAW,OAAO,QAAuB;AACrD,KAAI,IAAI,SAAS,WACf,SAAQ,KAAK,EAAE;AAGjB,KAAI,IAAI,SAAS,SAAS;EACxB,MAAM,EAAE,IAAI,WAAW,WAAW;AAElC,MAAI;GACF,MAAM,SAAsB;IAC1B;IACA,aAAa,EAAE,OAAO,SAAS,YAAY;AACzC,gBAAY,YAAY;MAAE,MAAM;MAAY;MAAI;MAAO;MAAS;MAAO,CAAkC;;IAE5G;GAED,MAAM,KAAK,MAAM,aAAa;IAC5B,QAAQ,OAAO;KACb,MAAM,OAAO;KACb,YAAY,gBAAA;KACb,CAAC;IACF,UAAU,aAAA;IACX,CAAC;AAEF,SAAM,GAAG,MAAM,WAAW,EAAE,YAAY,OAAO,YAAY,CAAC;AAC5D,SAAM,GAAG,SAAS;AAElB,cAAY,YAAY;IAAE,MAAM;IAAQ;IAAI,CAA8B;WAErE,KAAK;AACV,cAAY,YAAY;IACtB,MAAM;IACN;IACA,SAAS,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAA;IACtD,CAA+B;;;EAGpC"}
|
package/dist/sources/index.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { A as
|
|
2
|
-
export { DOC_OVERRIDES, type DocOverride, type EntryFile, type FetchedDoc, type GitDocsResult, type GitHubDiscussion, type GitHubIssue, type GitHubRelease, type LlmsContent, type LlmsLink, type LocalDependency, type LocalPackageInfo, type NpmPackageInfo, type ResolveAttempt, type ResolveOptions, type ResolveResult, type ResolveStep, type ResolvedPackage, downloadLlmsDocs, extractSections, fetchGitDocs, fetchGitHubDiscussions, fetchGitHubIssues, fetchGitHubRepoMeta, fetchLatestVersion, fetchLlmsTxt, fetchLlmsUrl, fetchNpmPackage, fetchNpmRegistryMeta, fetchPkgDist, fetchReadme, fetchReadmeContent, fetchReleaseNotes, fetchText,
|
|
1
|
+
import { $ as NpmPackageInfo, A as downloadLlmsDocs, B as isGhAvailable, C as readLocalDependencies, D as resolvePackageDocs, E as resolveLocalPackageDocs, F as parseMarkdownLinks, G as fetchReadme, H as MIN_GIT_DOCS, I as GitHubIssue, J as validateGitDocsWithLlms, K as fetchReadmeContent, L as fetchGitHubIssues, M as fetchLlmsTxt, N as fetchLlmsUrl, O as resolvePackageDocsWithAttempts, P as normalizeLlmsLinks, Q as LocalDependency, R as formatIssueAsMarkdown, S as parseVersionSpecifier, T as resolveInstalledVersion, U as fetchGitDocs, V as GitDocsResult, W as fetchGitHubRepoMeta, X as LlmsContent, Y as FetchedDoc, Z as LlmsLink, _ as fetchLatestVersion, a as normalizeRepoUrl, at as GitHubDiscussion, b as fetchPkgDist, c as GitHubRelease, ct as generateDiscussionIndex, d as DOC_OVERRIDES, et as ResolveAttempt, f as DocOverride, g as ResolveStep, h as ResolveOptions, i as isGitHubRepoUrl, it as resolveEntryFiles, j as extractSections, k as searchNpmPackages, l as fetchReleaseNotes, m as LocalPackageInfo, n as extractBranchHint, nt as ResolvedPackage, o as parseGitHubUrl, ot as fetchGitHubDiscussions, p as getDocOverride, q as isShallowGitDocs, r as fetchText, rt as EntryFile, s as verifyUrl, st as formatDiscussionAsMarkdown, t as $fetch, tt as ResolveResult, u as generateReleaseIndex, v as fetchNpmPackage, w as readLocalPackageInfo, x as getInstalledSkillVersion, y as fetchNpmRegistryMeta, z as generateIssueIndex } from "../_chunks/utils.mjs";
|
|
2
|
+
export { $fetch, DOC_OVERRIDES, type DocOverride, type EntryFile, type FetchedDoc, type GitDocsResult, type GitHubDiscussion, type GitHubIssue, type GitHubRelease, type LlmsContent, type LlmsLink, type LocalDependency, type LocalPackageInfo, MIN_GIT_DOCS, type NpmPackageInfo, type ResolveAttempt, type ResolveOptions, type ResolveResult, type ResolveStep, type ResolvedPackage, downloadLlmsDocs, extractBranchHint, extractSections, fetchGitDocs, fetchGitHubDiscussions, fetchGitHubIssues, fetchGitHubRepoMeta, fetchLatestVersion, fetchLlmsTxt, fetchLlmsUrl, fetchNpmPackage, fetchNpmRegistryMeta, fetchPkgDist, fetchReadme, fetchReadmeContent, fetchReleaseNotes, fetchText, formatDiscussionAsMarkdown, formatIssueAsMarkdown, generateDiscussionIndex, generateIssueIndex, generateReleaseIndex, getDocOverride, getInstalledSkillVersion, isGhAvailable, isGitHubRepoUrl, isShallowGitDocs, normalizeLlmsLinks, normalizeRepoUrl, parseGitHubUrl, parseMarkdownLinks, parseVersionSpecifier, readLocalDependencies, readLocalPackageInfo, resolveEntryFiles, resolveInstalledVersion, resolveLocalPackageDocs, resolvePackageDocs, resolvePackageDocsWithAttempts, searchNpmPackages, validateGitDocsWithLlms, verifyUrl };
|
package/dist/sources/index.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import "../_chunks/config.mjs";
|
|
2
|
-
import { A as
|
|
3
|
-
export { DOC_OVERRIDES, downloadLlmsDocs, extractSections, fetchGitDocs, fetchGitHubDiscussions, fetchGitHubIssues, fetchGitHubRepoMeta, fetchLatestVersion, fetchLlmsTxt, fetchLlmsUrl, fetchNpmPackage, fetchNpmRegistryMeta, fetchPkgDist, fetchReadme, fetchReadmeContent, fetchReleaseNotes, fetchText,
|
|
2
|
+
import { A as extractBranchHint, B as formatDiscussionAsMarkdown, C as fetchGitDocs, D as isShallowGitDocs, E as fetchReadmeContent, F as verifyUrl, G as isGhAvailable, H as fetchGitHubIssues, I as DOC_OVERRIDES, L as getDocOverride, M as isGitHubRepoUrl, N as normalizeRepoUrl, O as validateGitDocsWithLlms, P as parseGitHubUrl, R as resolveEntryFiles, S as MIN_GIT_DOCS, T as fetchReadme, U as formatIssueAsMarkdown, V as generateDiscussionIndex, W as generateIssueIndex, _ as extractSections, a as fetchNpmRegistryMeta, b as normalizeLlmsLinks, c as parseVersionSpecifier, d as resolveInstalledVersion, f as resolveLocalPackageDocs, g as downloadLlmsDocs, h as searchNpmPackages, i as fetchNpmPackage, j as fetchText, k as $fetch, l as readLocalDependencies, m as resolvePackageDocsWithAttempts, n as generateReleaseIndex, o as fetchPkgDist, p as resolvePackageDocs, r as fetchLatestVersion, s as getInstalledSkillVersion, t as fetchReleaseNotes, u as readLocalPackageInfo, v as fetchLlmsTxt, w as fetchGitHubRepoMeta, x as parseMarkdownLinks, y as fetchLlmsUrl, z as fetchGitHubDiscussions } from "../_chunks/releases.mjs";
|
|
3
|
+
export { $fetch, DOC_OVERRIDES, MIN_GIT_DOCS, downloadLlmsDocs, extractBranchHint, extractSections, fetchGitDocs, fetchGitHubDiscussions, fetchGitHubIssues, fetchGitHubRepoMeta, fetchLatestVersion, fetchLlmsTxt, fetchLlmsUrl, fetchNpmPackage, fetchNpmRegistryMeta, fetchPkgDist, fetchReadme, fetchReadmeContent, fetchReleaseNotes, fetchText, formatDiscussionAsMarkdown, formatIssueAsMarkdown, generateDiscussionIndex, generateIssueIndex, generateReleaseIndex, getDocOverride, getInstalledSkillVersion, isGhAvailable, isGitHubRepoUrl, isShallowGitDocs, normalizeLlmsLinks, normalizeRepoUrl, parseGitHubUrl, parseMarkdownLinks, parseVersionSpecifier, readLocalDependencies, readLocalPackageInfo, resolveEntryFiles, resolveInstalledVersion, resolveLocalPackageDocs, resolvePackageDocs, resolvePackageDocsWithAttempts, searchNpmPackages, validateGitDocsWithLlms, verifyUrl };
|
package/dist/types.d.mts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
1
|
+
import { D as CacheConfig, k as CachedPackage } from "./_chunks/version.mjs";
|
|
2
|
+
import { c as SearchResult, l as SearchSnippet, n as Document, o as SearchFilter, r as IndexConfig, s as SearchOptions } from "./_chunks/types.mjs";
|
|
3
|
+
import "./retriv/index.mjs";
|
|
4
|
+
import { X as LlmsContent, Y as FetchedDoc, Z as LlmsLink } from "./_chunks/utils.mjs";
|
|
5
|
+
import "./sources/index.mjs";
|
|
4
6
|
export { type CacheConfig, type CachedPackage, type Document, type FetchedDoc, type IndexConfig, type LlmsContent, type LlmsLink, type SearchFilter, type SearchOptions, type SearchResult, type SearchSnippet };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "skilld",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.2.0",
|
|
5
5
|
"description": "Generate AI agent skills from npm package documentation",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"skills"
|
|
41
41
|
],
|
|
42
42
|
"engines": {
|
|
43
|
-
"node": ">=
|
|
43
|
+
"node": ">=22.6.0"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
46
|
"@clack/prompts": "^1.0.0",
|
|
@@ -50,19 +50,23 @@
|
|
|
50
50
|
"globby": "^16.1.0",
|
|
51
51
|
"log-update": "^7.1.0",
|
|
52
52
|
"mlly": "^1.8.0",
|
|
53
|
+
"ofetch": "^1.5.1",
|
|
53
54
|
"oxc-parser": "^0.112.0",
|
|
54
55
|
"p-limit": "^7.3.0",
|
|
55
|
-
"
|
|
56
|
+
"pathe": "^2.0.3",
|
|
57
|
+
"retriv": "^0.9.1",
|
|
56
58
|
"sqlite-vec": "^0.1.6",
|
|
57
|
-
"typescript": "^5.9.3"
|
|
59
|
+
"typescript": "^5.9.3",
|
|
60
|
+
"unagent": "^0.0.7"
|
|
58
61
|
},
|
|
59
62
|
"devDependencies": {
|
|
60
|
-
"@antfu/eslint-config": "^7.
|
|
61
|
-
"@types/node": "^25.2.
|
|
63
|
+
"@antfu/eslint-config": "^7.3.0",
|
|
64
|
+
"@types/node": "^25.2.2",
|
|
62
65
|
"@vitest/coverage-v8": "^4.0.18",
|
|
63
66
|
"bumpp": "^10.4.1",
|
|
64
67
|
"evalite": "^0.19.0",
|
|
65
68
|
"obuild": "^0.4.27",
|
|
69
|
+
"tsx": "^4.21.0",
|
|
66
70
|
"vitest": "^4.0.18"
|
|
67
71
|
},
|
|
68
72
|
"scripts": {
|
|
@@ -73,6 +77,6 @@
|
|
|
73
77
|
"typecheck": "tsc --noEmit",
|
|
74
78
|
"test": "vitest",
|
|
75
79
|
"test:run": "vitest run",
|
|
76
|
-
"release": "
|
|
80
|
+
"release": "pnpm build && bumpp -x \"npx changelogen --output=CHANGELOG.md\""
|
|
77
81
|
}
|
|
78
82
|
}
|