skilld 0.13.3 → 0.13.5

package/dist/_chunks/{embedding-cache.mjs → embedding-cache2.mjs}

@@ -1,8 +1,11 @@
-import { n as __require } from "./chunk.mjs";
+import { n as __require, t as __exportAll } from "./chunk.mjs";
 import { t as CACHE_DIR } from "./config.mjs";
 import { join } from "pathe";
 import { rmSync } from "node:fs";
-import { cachedEmbeddings } from "retriv/embeddings/cached";
+var embedding_cache_exports = /* @__PURE__ */ __exportAll({
+	cachedEmbeddings: () => cachedEmbeddings,
+	clearEmbeddingCache: () => clearEmbeddingCache
+});
 const EMBEDDINGS_DB_PATH = join(CACHE_DIR, "embeddings.db");
 function openDb() {
 	const { DatabaseSync: DB } = __require("node:sqlite");
@@ -28,11 +31,12 @@ function createSqliteStorage(db) {
 		}
 	};
 }
-function cachedEmbeddings$1(config) {
+async function cachedEmbeddings(config) {
+	const { cachedEmbeddings: retrivCached } = await import("retriv/embeddings/cached");
 	const db = openDb();
 	const storage = createSqliteStorage(db);
 	const originalResolve = config.resolve;
-	return cachedEmbeddings({ async resolve() {
+	return retrivCached({ async resolve() {
 		const resolved = await originalResolve();
 		const getMetaStmt = db.prepare("SELECT value FROM meta WHERE key = ?");
 		const setMetaStmt = db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
@@ -45,6 +49,6 @@ function cachedEmbeddings$1(config) {
 function clearEmbeddingCache() {
 	rmSync(EMBEDDINGS_DB_PATH, { force: true });
 }
-export { clearEmbeddingCache as n, cachedEmbeddings$1 as t };
+export { clearEmbeddingCache as n, embedding_cache_exports as r, cachedEmbeddings as t };
 
-//# sourceMappingURL=embedding-cache.mjs.map
+//# sourceMappingURL=embedding-cache2.mjs.map

package/dist/_chunks/embedding-cache2.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-cache2.mjs","names":[],"sources":["../../src/retriv/embedding-cache.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite'\nimport type { Embedding } from 'retriv'\nimport { rmSync } from 'node:fs'\nimport { join } from 'pathe'\nimport { CACHE_DIR } from '../cache/index.ts'\n\ninterface EmbeddingConfig {\n  resolve: () => Promise<{ embedder: (texts: string[]) => Promise<Embedding[]>, dimensions: number, maxTokens?: number }>\n}\n\nconst EMBEDDINGS_DB_PATH = join(CACHE_DIR, 'embeddings.db')\n\nfunction openDb(): DatabaseSync {\n  // eslint-disable-next-line ts/no-require-imports\n  const { DatabaseSync: DB } = require('node:sqlite') as typeof import('node:sqlite')\n  const db = new DB(EMBEDDINGS_DB_PATH)\n  db.exec('PRAGMA journal_mode=WAL')\n  db.exec('PRAGMA busy_timeout=5000')\n  db.exec(`CREATE TABLE IF NOT EXISTS embeddings (text_hash TEXT PRIMARY KEY, embedding BLOB NOT NULL)`)\n  db.exec(`CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`)\n  return db\n}\n\nfunction createSqliteStorage(db: DatabaseSync) {\n  const getStmt = db.prepare('SELECT embedding FROM embeddings WHERE text_hash = ?')\n  const setStmt = db.prepare('INSERT OR IGNORE INTO embeddings (text_hash, embedding) VALUES (?, ?)')\n\n  return {\n    get: (hash: string): Embedding | null => {\n      const row = getStmt.get(hash) as { embedding: Buffer } | undefined\n      if (!row)\n        return null\n      return new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4)\n    },\n    set: (hash: string, embedding: Embedding): void => {\n      const arr = embedding instanceof Float32Array ? embedding : new Float32Array(embedding)\n      setStmt.run(hash, Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength))\n    },\n  }\n}\n\nexport async function cachedEmbeddings(config: EmbeddingConfig): Promise<EmbeddingConfig> {\n  const { cachedEmbeddings: retrivCached } = await import('retriv/embeddings/cached')\n  const db = openDb()\n  const storage = createSqliteStorage(db)\n\n  const originalResolve = config.resolve\n  const validatedConfig: EmbeddingConfig = {\n    async resolve() {\n      const resolved = await originalResolve()\n      const getMetaStmt = db.prepare('SELECT value FROM meta WHERE key = ?')\n      const setMetaStmt = db.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)')\n\n      const storedDims = getMetaStmt.get('dimensions') as { value: string } | undefined\n      if (storedDims && Number(storedDims.value) !== resolved.dimensions) {\n        db.exec('DELETE FROM embeddings')\n      }\n      setMetaStmt.run('dimensions', String(resolved.dimensions))\n\n      return resolved\n    },\n  }\n\n  return retrivCached(validatedConfig, { storage })\n}\n\nexport function clearEmbeddingCache(): void {\n  rmSync(EMBEDDINGS_DB_PATH, { force: true })\n}\n"],"mappings":";;;;;;;;AAUA,MAAM,qBAAqB,KAAK,WAAW,gBAAgB;AAE3D,SAAS,SAAuB;CAE9B,MAAM,EAAE,cAAc,OAAA,UAAe,cAAc;CACnD,MAAM,KAAK,IAAI,GAAG,mBAAmB;AACrC,IAAG,KAAK,0BAA0B;AAClC,IAAG,KAAK,2BAA2B;AACnC,IAAG,KAAK,8FAA8F;AACtG,IAAG,KAAK,8EAA8E;AACtF,QAAO;;AAGT,SAAS,oBAAoB,IAAkB;CAC7C,MAAM,UAAU,GAAG,QAAQ,uDAAuD;CAClF,MAAM,UAAU,GAAG,QAAQ,wEAAwE;AAEnG,QAAO;EACL,MAAM,SAAmC;GACvC,MAAM,MAAM,QAAQ,IAAI,KAAK;AAC7B,OAAI,CAAC,IACH,QAAO;AACT,UAAO,IAAI,aAAa,IAAI,UAAU,QAAQ,IAAI,UAAU,YAAY,IAAI,UAAU,aAAa,EAAE;;EAEvG,MAAM,MAAc,cAA+B;GACjD,MAAM,MAAM,qBAAqB,eAAe,YAAY,IAAI,aAAa,UAAU;AACvF,WAAQ,IAAI,MAAM,OAAO,KAAK,IAAI,QAAQ,IAAI,YAAY,IAAI,WAAW,CAAC;;EAE7E;;AAGH,eAAsB,iBAAiB,QAAmD;CACxF,MAAM,EAAE,kBAAkB,iBAAiB,MAAM,OAAO;CACxD,MAAM,KAAK,QAAQ;CACnB,MAAM,UAAU,oBAAoB,GAAG;CAEvC,MAAM,kBAAkB,OAAO;AAiB/B,QAAO,aAhBkC,EACvC,MAAM,UAAU;EACd,MAAM,WAAW,MAAM,iBAAiB;EACxC,MAAM,cAAc,GAAG,QAAQ,uCAAuC;EACtE,MAAM,cAAc,GAAG,QAAQ,yDAAyD;EAExF,MAAM,aAAa,YAAY,IAAI,aAAa;AAChD,MAAI,cAAc,OAAO,WAAW,MAAM,KAAK,SAAS,WACtD,IAAG,KAAK,yBAAyB;AAEnC,cAAY,IAAI,cAAc,OAAO,SAAS,WAAW,CAAC;AAE1D,SAAO;IAEV,EAEoC,EAAE,SAAS,CAAC;;AAGnD,SAAgB,sBAA4B;AAC1C,QAAO,oBAAoB,EAAE,OAAO,MAAM,CAAC"}

package/dist/_chunks/npm.mjs

@@ -10,8 +10,9 @@ import { gt } from "semver";
 import { ofetch } from "ofetch";
 import { crawlAndGenerate } from "@mdream/crawl";
 import { globby } from "globby";
-import pLimit from "p-limit";
+import { downloadTemplate } from "giget";
 import { fileURLToPath, pathToFileURL } from "node:url";
+import pLimit from "p-limit";
 import { Writable } from "node:stream";
 import { resolvePathSync } from "mlly";
 const BOT_USERS = new Set([
@@ -1119,11 +1120,20 @@ function parseSkillFrontmatterName(content) {
 		description: fm.description
 	};
 }
-const SUPPORTING_DIRS = [
-	"scripts",
-	"references",
-	"assets"
-];
+function collectFiles(dir, prefix = "") {
+	const files = [];
+	if (!existsSync(dir)) return files;
+	for (const entry of readdirSync(dir, { withFileTypes: true })) {
+		const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
+		const fullPath = resolve(dir, entry.name);
+		if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
+		else if (entry.isFile()) files.push({
+			path: relPath,
+			content: readFileSync(fullPath, "utf-8")
+		});
+	}
+	return files;
+}
 async function fetchGitSkills(source, onProgress) {
 	if (source.type === "local") return fetchLocalSkills(source);
 	if (source.type === "github") return fetchGitHubSkills(source, onProgress);
@@ -1153,18 +1163,7 @@ function readLocalSkill(dir, repoPath) {
 	const frontmatter = parseSkillFrontmatterName(content);
 	const dirName = dir.split("/").pop();
 	const name = frontmatter.name || dirName;
-	const files = [];
-	for (const subdir of SUPPORTING_DIRS) {
-		const subdirPath = resolve(dir, subdir);
-		if (!existsSync(subdirPath)) continue;
-		for (const file of readdirSync(subdirPath, { withFileTypes: true })) {
-			if (!file.isFile()) continue;
-			files.push({
-				path: `${subdir}/${file.name}`,
-				content: readFileSync(resolve(subdirPath, file.name), "utf-8")
-			});
-		}
-	}
+	const files = collectFiles(dir).filter((f) => f.path !== "SKILL.md");
 	return {
 		name,
 		description: frontmatter.description || "",
@@ -1177,120 +1176,110 @@ async function fetchGitHubSkills(source, onProgress) {
 	const { owner, repo } = source;
 	if (!owner || !repo) return { skills: [] };
 	const ref = source.ref || "main";
-	onProgress?.(`Listing files at ${owner}/${repo}@${ref}`);
-	const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
-	if (!data?.files?.length) {
-		if (ref === "main") {
-			const fallback = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/master`).catch(() => null);
-			if (fallback?.files?.length) return extractGitHubSkills(owner, repo, "master", fallback, source.skillPath, onProgress);
-		}
-		return { skills: [] };
+	const refs = ref === "main" ? ["main", "master"] : [ref];
+	for (const tryRef of refs) {
+		const skills = await downloadGitHubSkills(owner, repo, tryRef, source.skillPath, onProgress);
+		if (skills.length > 0) return { skills };
 	}
-	return extractGitHubSkills(owner, repo, ref, data, source.skillPath, onProgress);
-}
-async function extractGitHubSkills(owner, repo, ref, data, skillPath, onProgress) {
-	const allFiles = data.files.map((f) => f.path);
-	const commitSha = data.meta?.sha;
-	let skillMdPaths;
-	if (skillPath) {
-		const candidates = [`${skillPath}/SKILL.md`, skillPath.endsWith("/SKILL.md") ? skillPath : null].filter(Boolean);
-		skillMdPaths = allFiles.filter((f) => candidates.includes(f));
-	} else skillMdPaths = allFiles.filter((f) => f.match(/^skills\/[^/]+\/SKILL\.md$/) || f === "SKILL.md");
-	if (skillMdPaths.length === 0) return {
-		skills: [],
-		commitSha
-	};
-	const limit = pLimit(5);
-	const skills = [];
-	onProgress?.(`Found ${skillMdPaths.length} skill(s), downloading...`);
-	await Promise.all(skillMdPaths.map((mdPath) => limit(async () => {
-		const skillDir = mdPath === "SKILL.md" ? "" : mdPath.replace(/\/SKILL\.md$/, "");
-		const content = await fetchRawGitHub(owner, repo, ref, mdPath);
-		if (!content) return;
-		const frontmatter = parseSkillFrontmatterName(content);
-		const dirName = skillDir ? skillDir.split("/").pop() : repo;
-		const name = frontmatter.name || dirName;
-		const supportingFiles = [];
-		const prefix = skillDir ? `${skillDir}/` : "";
-		for (const subdir of SUPPORTING_DIRS) {
-			const subdirPrefix = `${prefix}${subdir}/`;
-			const matching = allFiles.filter((f) => f.startsWith(subdirPrefix));
-			for (const filePath of matching) {
-				const fileContent = await fetchRawGitHub(owner, repo, ref, filePath);
-				if (fileContent) {
-					const relativePath = filePath.slice(prefix.length);
-					supportingFiles.push({
-						path: relativePath,
-						content: fileContent
-					});
-				}
+	return { skills: [] };
+}
+async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
+	const tempDir = join(tmpdir(), `skilld-${Date.now()}`);
+	try {
+		if (skillPath) {
+			onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
+			const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
+				dir: tempDir,
+				force: true
+			});
+			const skill = readLocalSkill(dir, skillPath);
+			return skill ? [skill] : [];
+		}
+		onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
+		try {
+			const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
+				dir: tempDir,
+				force: true
+			});
+			const skills = [];
+			for (const entry of readdirSync(dir, { withFileTypes: true })) {
+				if (!entry.isDirectory()) continue;
+				const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
+				if (skill) skills.push(skill);
+			}
+			if (skills.length > 0) {
+				onProgress?.(`Found ${skills.length} skill(s)`);
+				return skills;
 			}
+		} catch {}
+		const content = await $fetch(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
+		if (content) {
+			const fm = parseSkillFrontmatterName(content);
+			onProgress?.("Found 1 skill");
+			return [{
+				name: fm.name || repo,
+				description: fm.description || "",
+				path: "",
+				content,
+				files: []
+			}];
 		}
-		skills.push({
-			name,
-			description: frontmatter.description || "",
-			path: skillDir,
-			content,
-			files: supportingFiles
+		return [];
+	} catch {
+		return [];
+	} finally {
+		rmSync(tempDir, {
+			recursive: true,
+			force: true
 		});
-	})));
-	return {
-		skills,
-		commitSha
-	};
-}
-async function fetchRawGitHub(owner, repo, ref, path) {
-	return $fetch(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${path}`, { responseType: "text" }).catch(() => null);
+	}
 }
 async function fetchGitLabSkills(source, onProgress) {
 	const { owner, repo } = source;
 	if (!owner || !repo) return { skills: [] };
 	const ref = source.ref || "main";
-	const projectId = encodeURIComponent(`${owner}/${repo}`);
-	onProgress?.(`Listing files at ${owner}/${repo}@${ref}`);
-	const tree = await $fetch(`https://gitlab.com/api/v4/projects/${projectId}/repository/tree?ref=${ref}&recursive=true&per_page=100`).catch(() => null);
-	if (!tree?.length) return { skills: [] };
-	const allFiles = tree.filter((e) => e.type === "blob").map((e) => e.path);
-	const skillMdPaths = source.skillPath ? allFiles.filter((f) => f === `${source.skillPath}/SKILL.md`) : allFiles.filter((f) => f.match(/^skills\/[^/]+\/SKILL\.md$/) || f === "SKILL.md");
-	if (skillMdPaths.length === 0) return { skills: [] };
-	const limit = pLimit(5);
-	const skills = [];
-	onProgress?.(`Found ${skillMdPaths.length} skill(s), downloading...`);
-	await Promise.all(skillMdPaths.map((mdPath) => limit(async () => {
-		const skillDir = mdPath === "SKILL.md" ? "" : mdPath.replace(/\/SKILL\.md$/, "");
-		const content = await fetchRawGitLab(owner, repo, ref, mdPath);
-		if (!content) return;
-		const frontmatter = parseSkillFrontmatterName(content);
-		const dirName = skillDir ? skillDir.split("/").pop() : repo;
-		const name = frontmatter.name || dirName;
-		const supportingFiles = [];
-		const prefix = skillDir ? `${skillDir}/` : "";
-		for (const subdir of SUPPORTING_DIRS) {
-			const subdirPrefix = `${prefix}${subdir}/`;
-			const matching = allFiles.filter((f) => f.startsWith(subdirPrefix));
-			for (const filePath of matching) {
-				const fileContent = await fetchRawGitLab(owner, repo, ref, filePath);
-				if (fileContent) {
-					const relativePath = filePath.slice(prefix.length);
-					supportingFiles.push({
-						path: relativePath,
-						content: fileContent
-					});
-				}
-			}
+	const tempDir = join(tmpdir(), `skilld-gitlab-${Date.now()}`);
+	try {
+		const subdir = source.skillPath || "skills";
+		onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
+		const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
+			dir: tempDir,
+			force: true
+		});
+		if (source.skillPath) {
+			const skill = readLocalSkill(dir, source.skillPath);
+			return { skills: skill ? [skill] : [] };
 		}
-		skills.push({
-			name,
-			description: frontmatter.description || "",
-			path: skillDir,
-			content,
-			files: supportingFiles
+		const skills = [];
+		for (const entry of readdirSync(dir, { withFileTypes: true })) {
+			if (!entry.isDirectory()) continue;
+			const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
+			if (skill) skills.push(skill);
+		}
+		if (skills.length > 0) {
+			onProgress?.(`Found ${skills.length} skill(s)`);
+			return { skills };
+		}
+		const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
+		if (content) {
+			const fm = parseSkillFrontmatterName(content);
+			return { skills: [{
+				name: fm.name || repo,
+				description: fm.description || "",
+				path: "",
+				content,
+				files: []
+			}] };
+		}
+		return { skills: [] };
+	} catch {
+		return { skills: [] };
+	} finally {
+		rmSync(tempDir, {
+			recursive: true,
+			force: true
 		});
-	})));
-	return { skills };
-}
-async function fetchRawGitLab(owner, repo, ref, path) {
-	return $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/${path}`, { responseType: "text" }).catch(() => null);
+	}
 }
 async function fetchLlmsUrl(docsUrl) {
 	const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;