toolnet-memory 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/toolnet-memory +44 -0
- package/bundle/agy-hook.js +39 -88
- package/bundle/agy.js +38 -104
- package/bundle/auto-integrate.js +131 -707
- package/bundle/codex-context.js +44 -54
- package/bundle/codex.js +47 -83
- package/bundle/context-runtime.js +29 -124
- package/bundle/context.js +14 -15
- package/bundle/doctor.js +3 -6
- package/bundle/full-index.js +8 -18
- package/bundle/graph-index.js +4 -4
- package/bundle/impact.js +3 -3
- package/bundle/incremental.js +4 -4
- package/bundle/learner.js +5 -5
- package/bundle/mcp.js +7 -54
- package/bundle/opencode.js +168 -717
- package/bundle/project-manual.js +6 -6
- package/bundle/runtime.js +11 -11
- package/bundle/semantic.js +6 -6
- package/bundle/setup.js +187 -716
- package/bundle/snapshot.js +4 -4
- package/bundle/update.js +1 -4
- package/bundle/work-continuity.js +6 -6
- package/package.json +1 -1
- package/bundle/api.js +0 -52
- package/bundle/claude-hook.js +0 -78
- package/bundle/claude.js +0 -3
- package/bundle/continuity-certify.js +0 -49
- package/bundle/handoff-refresh.js +0 -12
- package/bundle/init.js +0 -763
- package/bundle/memory-agent.js +0 -48
- package/bundle/memory-query.js +0 -7
- package/bundle/production-certify.js +0 -5
- package/bundle/provider.js +0 -3
- package/bundle/recovery-certify.js +0 -55
package/bundle/semantic.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import"dotenv/config";import{existsSync as
|
|
2
|
-
`).replace(/\\r/g,"\r").replace(/\\t/g," ").replace(/\\"/g,'"').replace(/\\\\/g,"\\")):e.length>=2&&e.startsWith("'")&&e.endsWith("'")?e.slice(1,-1):e}function
|
|
3
|
-
`,{encoding:"utf8",mode:384}),De(o,s)}function pe(r,e){return{id:r.id,name:r.name,remote:r.remote,rootPath:e,createdAt:r.createdAt,updatedAt:r.updatedAt,graphVersion:r.graphVersion,memoryVersion:r.memoryVersion,metadata:r.metadata}}var F=class{detect(e=process.cwd()){let t=_(e),s=ze(t),i=[".git","package.json","pyproject.toml","Cargo.toml","go.mod","composer.json"].some(p=>te(K(s,p))),a=Be(t,i?s:void 0);if(a){let p=re(a),f=He(p);return f.rootPath!==a&&(f.rootPath=a,f.updatedAt=new Date().toISOString(),de(a,f)),pe(f,a)}let c=new Date().toISOString(),l=Ke(s),d={version:1,id:Ge(s),name:l,remote:l,rootPath:s,createdAt:c,updatedAt:c,graphVersion:0,memoryVersion:0};return de(s,d),pe(d,s)}};var Ve=[{type:"openai_key",regex:/\bsk-[A-Za-z0-9_\-]{20,}\b/g},{type:"huggingface_token",regex:/\bhf_[A-Za-z0-9]{20,}\b/g},{type:"hf_s3_access_key",regex:/\bHFAK[A-Za-z0-9]{8,}\b/g},{type:"bearer_token",regex:/\bBearer\s+[A-Za-z0-9._\-]{16,}\b/gi},{type:"jwt",regex:/\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\b/g},{type:"private_key",regex:/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g},{type:"password_assignment",regex:/\b(password|passwd|pwd)\s*[:=]\s*["']?[^"'\s]{6,}["']?/gi},{type:"secret_assignment",regex:/\b(secret|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*["']?[^"'\s]{8,}["']?/gi},{type:"cookie",regex:/\b(cookie|set-cookie)\s*[:=]\s*[^;\n]{8,}/gi}],$=class{scan(e){let t=[];for(let s of Ve){let o=new RegExp(s.regex.source,s.regex.flags);for(let i of e.matchAll(o))t.push({type:s.type,value:i[0]})}return t}hasSecrets(e){return this.scan(e).length>0}};var M=class{scanner=new $;sanitize(e){let t=e,s=this.scanner.scan(e),o=new Set;for(let i of s)o.add(i.type),t=t.split(i.value).join(`[REDACTED:${i.type}]`);return{text:t,redacted:s.length,secretTypes:[...o]}}sanitizeValue(e){if(typeof e=="string")return this.sanitize(e).text;if(Array.isArray(e))return e.map(t=>this.sanitizeValue(t));if(e&&typeof e=="object"){let t={};for(let[s,o]of Object.entries(e)){let i=s.toLowerCase();i.includes("password")||i.includes("secret")||i.includes("token")||i.includes("cookie")||i.includes("authorization")?t[s]="[REDACTED]":t[s]=this.sanitizeValue(o)}return t}return e}};import{homedir as gt}from"node:os";import{join as ht}from"node:path";import{DeleteObjectCommand as We,GetObjectCommand as Ye,HeadObjectCommand as qe,ListObjectsV2Command as Je,PutObjectCommand as Qe,S3Client as Ze}from"@aws-sdk/client-s3";import{getSignedUrl as Xe}from"@aws-sdk/s3-request-presigner";var G=class{name="huggingface";client;bucket;constructor(e){this.bucket=e.bucket,this.client=new Ze({region:"us-east-1",endpoint:`https://s3.hf.co/${e.namespace}`,forcePathStyle:!0,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,s="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new Qe({Bucket:this.bucket,Key:e,Body:o,ContentType:s}))}async get(e){let t=await Xe(this.client,new Ye({Bucket:this.bucket,Key:e}),{expiresIn:60}),s=await fetch(t,{redirect:"follow"});if(s.status===404)return null;if(!s.ok)throw new Error(`HF download failed: ${s.status} ${s.statusText}`);return new Uint8Array(await s.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new qe({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new We({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],s;do{let o=await this.client.send(new Je({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:s}));for(let i of o.Contents??[])i.Key&&t.push({key:i.Key,size:i.Size,updatedAt:i.LastModified?.toISOString()});s=o.IsTruncated?o.NextContinuationToken:void 0}while(s);return t}};import{access as fe,mkdir as et,readFile as tt,readdir as rt,rm as st,stat as ge,writeFile as ot}from"node:fs/promises";import{dirname as nt,join as it,relative as he,resolve as at}from"node:path";var N=class{constructor(e){this.root=e}root;name="local";path(e){let t=e.replace(/^\/+/,"");return at(this.root,t)}async put(e,t){let s=this.path(e);await et(nt(s),{recursive:!0}),await ot(s,t)}async get(e){try{return await tt(this.path(e))}catch(t){if(typeof t=="object"&&t!==null&&"code"in t&&t.code==="ENOENT")return null;throw t}}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await fe(this.path(e)),!0}catch{return!1}}async delete(e){await st(this.path(e),{force:!0})}async list(e=""){let t=this.path(e),s=[];try{await fe(t)}catch{return s}let o=async a=>{let c=await rt(a,{withFileTypes:!0});for(let l of c){let d=it(a,l.name);if(l.isDirectory()){await o(d);continue}let p=await ge(d);s.push({key:he(this.root,d),size:p.size,updatedAt:p.mtime.toISOString()})}},i=await ge(t);return i.isDirectory()?await o(t):s.push({key:he(this.root,t),size:i.size,updatedAt:i.mtime.toISOString()}),s}};import{DeleteObjectCommand as ct,GetObjectCommand as lt,HeadObjectCommand as dt,ListObjectsV2Command as pt,PutObjectCommand as ut,S3Client as mt}from"@aws-sdk/client-s3";import{getSignedUrl as ft}from"@aws-sdk/s3-request-presigner";var O=class{name;client;bucket;constructor(e){this.name=e.name??"s3",this.bucket=e.bucket,this.client=new mt({region:e.region??"us-east-1",endpoint:e.endpoint||void 0,forcePathStyle:e.forcePathStyle??!1,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,s="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new ut({Bucket:this.bucket,Key:e,Body:o,ContentType:s}))}async get(e){let t=await ft(this.client,new lt({Bucket:this.bucket,Key:e}),{expiresIn:60}),s=await fetch(t,{redirect:"follow"});if(s.status===404)return null;if(!s.ok)throw new Error(`${this.name} download failed: ${s.status} ${s.statusText}`);return new Uint8Array(await s.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new dt({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new ct({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],s;do{let o=await this.client.send(new pt({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:s}));for(let i of o.Contents??[])i.Key&&t.push({key:i.Key,size:i.Size,updatedAt:i.LastModified?.toISOString()});s=o.IsTruncated?o.NextContinuationToken:void 0}while(s);return t}};function se(r,e){return console.warn(e),new N(r)}function ye(r){let e=r.localRoot??ht(gt(),".toolnet-memory","storage");if(r.provider==="r2"){let t=r.r2;return t?.accountId&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new O({name:"r2",endpoint:`https://${t.accountId}.r2.cloudflarestorage.com`,region:"auto",bucket:t.bucket,forcePathStyle:!0,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):se(e,"[storage] Cloudflare R2 credentials missing. Using local fallback.")}if(r.provider==="s3"){let t=r.s3;return t?.bucket&&t.accessKeyId&&t.secretAccessKey?new O({name:"s3",endpoint:t.endpoint,region:t.region??"us-east-1",bucket:t.bucket,forcePathStyle:t.forcePathStyle??!1,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):se(e,"[storage] S3 credentials missing. Using local fallback.")}if(r.provider==="huggingface"){let t=r.huggingface;return t?.namespace&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new G({namespace:t.namespace,bucket:t.bucket,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):se(e,"[storage] Hugging Face credentials missing. Using local fallback.")}return new N(e)}var j=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"graph","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var U=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","chunks","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var B=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","vectors","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};function be(r){let e=r.trim().replace(/\s+/g,"_").replace(/[^A-Za-z0-9._-]/g,"_").replace(/_+/g,"_").replace(/^\.+|\.+$/g,"").slice(0,100);if(!e||e==="."||e==="..")throw new Error("Invalid project storage folder");return e}function Se(r){let e=r.replace(/^\/+/,"");if(e.startsWith("memories/"))return"memory/records/"+e.slice(9);if(e.startsWith("vectors/"))return"memory/vectors/"+e.slice(8);if(e.startsWith("graph/"))return"code/graph/"+e.slice(6);let t=e.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);if(t)return`${t[1]}memory/records/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/),t)return`${t[1]}memory/vectors/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/),t)return`${t[1]}code/graph/${t[2]}`;let s=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);if(s)return`${s[1]}memory/records/${s[2]}`;if(s=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/),s)return`${s[1]}memory/vectors/${s[2]}`;if(s=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/),s)return`${s[1]}code/graph/${s[2]}`;let o=e.match(/^(projects\/[^/]+\/)memories\/(.+)$/);return o?`${o[1]}memory/records/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)vectors\/(.+)$/),o?`${o[1]}memory/vectors/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)graph\/(.+)$/),o?`${o[1]}code/graph/${o[2]}`:e))}var z=class{constructor(e,t,s,o){this.provider=e;this.name=e.name,this.projectId=t,this.projectName=s,this.folder=be(o??s),this.sourcePrefix=`projects/${t}`,this.targetPrefix=`projects/${this.folder}`}provider;name;folder;sourcePrefix;targetPrefix;projectId;projectName;registryPromise;async registerProject(){let e=`${this.targetPrefix}/project.json`,t=new Date().toISOString(),s=t,o=await this.provider.getText(e);if(o){let a;try{a=JSON.parse(o)}catch(c){throw new Error(`Invalid remote ToolNet project manifest at ${e}: ${c instanceof Error?c.message:String(c)}`)}if(typeof a.id=="string"&&a.id!==this.projectId)throw new Error(["ToolNet remote project namespace collision.",`Remote folder: ${this.targetPrefix}`,`Existing project id: ${a.id}`,`Current project id: ${this.projectId}`,"Refusing to mix data from two projects."].join(" "));typeof a.createdAt=="string"&&(s=a.createdAt)}let i={version:1,id:this.projectId,name:this.projectName,remote:this.folder,createdAt:s,updatedAt:t};await this.provider.put(e,JSON.stringify(i,null,2)+`
|
|
4
|
-
`,"application/json")}async ensureRegistered(){return this.registryPromise||(this.registryPromise=this.registerProject()),this.registryPromise}key(e){if(e=
|
|
5
|
-
`).trim();if(
|
|
6
|
-
`)),Le=await o.embedMany(Ae);for(let C=0;C<w.length;C++){let x=w[C],P=Le[C];if(!(!x||!P||P.length===0)){if(k===void 0&&(k=P.length),P.length!==k){console.warn(`[code-semantic] skip dimension mismatch: ${P.length} != ${k}`);continue}this.vectors.upsert({id:x.id,projectId:e,vector:P,metadata:{contentHash:x.contentHash,filePath:x.filePath,symbolName:x.symbolName,symbolType:x.symbolType,startLine:x.startLine,endLine:x.endLine}}),R++}}}let ee=this.vectors.exportProject(e),ce=R>0||v>0||!y||y.model!==a;return ce&&await l.save({version:1,projectId:e,model:a,dimensions:ee[0]?.vector.length??k??0,updatedAt:new Date().toISOString(),records:ee}),{chunks:d.length,vectorsLoaded:E,vectorsIndexed:R,vectorsRemoved:v,totalVectors:ee.length,savedChunks:f,savedVectors:ce}}async search(e,t=8){let s=await this.options.embeddings.embed(e),o=this.vectors.search(this.options.projectId,s,Math.max(t*4,20)),i=new Set(oe(e)),a=[];for(let c of o){let l=this.chunks.get(c.id);if(!l)continue;let d=new Set(oe([l.filePath,l.symbolName??"",l.symbolType??"",l.content].join(" "))),p=0;for(let v of i)d.has(v)&&p++;let f=i.size===0?0:p/i.size,y=Math.max(0,c.score),E=Ce(l.filePath,e),b=(y*.75+f*.25)*E;a.push({chunk:l,score:b,vectorScore:y,lexicalScore:f})}return a.sort((c,l)=>l.score-c.score).slice(0,t)}size(){return this.chunks.size}};import*as S from"typescript";import*as u from"typescript";var on=new Set([u.SyntaxKind.EqualsToken,u.SyntaxKind.PlusEqualsToken,u.SyntaxKind.MinusEqualsToken,u.SyntaxKind.AsteriskEqualsToken,u.SyntaxKind.SlashEqualsToken,u.SyntaxKind.PercentEqualsToken,u.SyntaxKind.QuestionQuestionEqualsToken,u.SyntaxKind.AmpersandAmpersandEqualsToken,u.SyntaxKind.BarBarEqualsToken]);var ve=[{id:"openai-compatible",label:"OpenAI-compatible",requiresApiKey:!0,requiresBaseUrl:!0,transport:"openai-compatible"},{id:"alibaba",label:"Alibaba / DashScope",requiresApiKey:!0,requiresBaseUrl:!0,transport:"openai-compatible"},{id:"openrouter",label:"OpenRouter",defaultBaseUrl:"https://openrouter.ai/api/v1",requiresApiKey:!0,transport:"openai-compatible"},{id:"groq",label:"Groq",defaultBaseUrl:"https://api.groq.com/openai/v1",requiresApiKey:!0,transport:"openai-compatible"},{id:"deepseek",label:"DeepSeek",defaultBaseUrl:"https://api.deepseek.com",defaultModel:"deepseek-v4-flash",requiresApiKey:!0,transport:"openai-compatible"},{id:"nvidia",label:"NVIDIA NIM",defaultBaseUrl:"https://integrate.api.nvidia.com/v1",defaultModel:"deepseek-ai/deepseek-v4-pro",requiresApiKey:!0,transport:"openai-compatible"},{id:"gemini",label:"Gemini",defaultBaseUrl:"https://generativelanguage.googleapis.com/v1beta",requiresApiKey:!0,transport:"gemini"},{id:"huggingface",label:"Hugging Face",defaultBaseUrl:"https://router.huggingface.co/v1",requiresApiKey:!0,transport:"openai-compatible"},{id:"ollama",label:"Ollama / Local",defaultBaseUrl:"http://127.0.0.1:11434/v1",requiresApiKey:!1,transport:"openai-compatible"},{id:"custom",label:"Custom endpoint",requiresApiKey:!1,requiresBaseUrl:!0,transport:"openai-compatible"},{id:"cloudflare",label:"Cloudflare Workers AI",requiresApiKey:!0,requiresAccountId:!0,transport:"cloudflare"}];function Y(r){let e=ve.find(t=>t.id===r);if(!e)throw new Error(`Unsupported AI provider: ${r}`);return e}function q(r){return ve.some(e=>e.id===r)}function n(r){return process.env[r]?.trim()||void 0}function m(...r){return r.find(e=>!!e?.trim())}function xe(){if(n("GROQ_API_KEY"))return"groq";if(n("DEEPSEEK_API_KEY"))return"deepseek";if(n("NVIDIA_API_KEY")||n("NVIDIA_NIM_API_KEY"))return"nvidia";if(n("OPENROUTER_API_KEY"))return"openrouter";if(n("ALIBABA_API_KEY")||n("DASHSCOPE_API_KEY"))return"alibaba";if(n("GEMINI_API_KEY")||n("GOOGLE_API_KEY"))return"gemini";if(n("CLOUDFLARE_API_TOKEN")&&n("CLOUDFLARE_ACCOUNT_ID"))return"cloudflare";if(n("HF_TOKEN"))return"huggingface";if(n("OLLAMA_MODEL")||n("OLLAMA_BASE_URL"))return"ollama"}function Lt(){let r=n("TOOLNET_LLM_PROVIDER");return r&&q(r)?r:xe()??"openai-compatible"}function ne(r){let e=Y(r);switch(r){case"alibaba":return{provider:r,apiKey:m(n("ALIBABA_API_KEY"),n("DASHSCOPE_API_KEY")),baseUrl:m(n("ALIBABA_BASE_URL"),n("DASHSCOPE_BASE_URL"),e.defaultBaseUrl),model:m(n("ALIBABA_MODEL"),n("DASHSCOPE_MODEL"))};case"openrouter":return{provider:r,apiKey:n("OPENROUTER_API_KEY"),baseUrl:m(n("OPENROUTER_BASE_URL"),e.defaultBaseUrl),model:n("OPENROUTER_MODEL")};case"groq":return{provider:r,apiKey:n("GROQ_API_KEY"),baseUrl:m(n("GROQ_BASE_URL"),e.defaultBaseUrl),model:n("GROQ_MODEL")};case"deepseek":return{provider:r,apiKey:n("DEEPSEEK_API_KEY"),baseUrl:m(n("DEEPSEEK_BASE_URL"),e.defaultBaseUrl),model:m(n("DEEPSEEK_MODEL"),e.defaultModel)};case"nvidia":return{provider:r,apiKey:m(n("NVIDIA_API_KEY"),n("NVIDIA_NIM_API_KEY")),baseUrl:m(n("NVIDIA_BASE_URL"),n("NVIDIA_NIM_BASE_URL"),e.defaultBaseUrl),model:m(n("NVIDIA_MODEL"),n("NVIDIA_NIM_MODEL"),e.defaultModel)};case"gemini":return{provider:r,apiKey:m(n("GEMINI_API_KEY"),n("GOOGLE_API_KEY")),baseUrl:m(n("GEMINI_BASE_URL"),e.defaultBaseUrl),model:n("GEMINI_MODEL")};case"huggingface":return{provider:r,apiKey:n("HF_TOKEN"),baseUrl:m(n("HF_INFERENCE_BASE_URL"),e.defaultBaseUrl),model:m(n("HF_LLM_MODEL"),n("HF_MODEL"))};case"ollama":return{provider:r,apiKey:n("OLLAMA_API_KEY"),baseUrl:m(n("OLLAMA_BASE_URL"),e.defaultBaseUrl),model:n("OLLAMA_MODEL")};case"cloudflare":return{provider:r,accountId:n("CLOUDFLARE_ACCOUNT_ID"),apiKey:n("CLOUDFLARE_API_TOKEN"),baseUrl:n("CLOUDFLARE_AI_BASE_URL"),model:n("CLOUDFLARE_MODEL")};case"custom":return{provider:r,apiKey:n("CUSTOM_AI_API_KEY"),baseUrl:n("CUSTOM_AI_BASE_URL"),model:n("CUSTOM_AI_MODEL")};default:return{provider:"openai-compatible",apiKey:m(n("OPENAI_API_KEY"),n("MODEL_API_KEY")),baseUrl:m(n("OPENAI_BASE_URL"),n("MODEL_BASE_URL")),model:m(n("OPENAI_MODEL"),n("MODEL_NAME"))}}}function Rt(){let r=n("TOOLNET_LLM_PROVIDER"),e=r&&q(r)?r:Lt(),t=Y(e),s=ne(e);return{provider:e,apiKey:m(n("TOOLNET_LLM_API_KEY"),s.apiKey),baseUrl:m(n("TOOLNET_LLM_BASE_URL"),s.baseUrl,t.defaultBaseUrl),model:m(n("TOOLNET_LLM_MODEL"),s.model,t.defaultModel),accountId:m(n("TOOLNET_LLM_ACCOUNT_ID"),s.accountId)}}function kt(){let r=n("TOOLNET_EMBEDDING_PROVIDER");return r==="local"?"local":r&&q(r)?r:n("HF_TOKEN")||n("HF_EMBEDDING_MODEL")?"huggingface":"local"}function It(){let r=kt();if(r==="local")return{provider:"local",model:m(n("TOOLNET_EMBEDDING_MODEL"),n("LOCAL_EMBEDDING_MODEL"))};let e=Y(r),t,s,o,i;switch(r){case"huggingface":t=n("HF_TOKEN"),s=n("HF_INFERENCE_BASE_URL"),o=n("HF_EMBEDDING_MODEL");break;case"openai-compatible":t=n("OPENAI_API_KEY"),s=n("OPENAI_BASE_URL"),o=n("OPENAI_EMBEDDING_MODEL");break;case"cloudflare":t=n("CLOUDFLARE_API_TOKEN"),s=n("CLOUDFLARE_AI_BASE_URL"),o=n("CLOUDFLARE_EMBEDDING_MODEL"),i=n("CLOUDFLARE_ACCOUNT_ID");break;default:t=ne(r).apiKey,s=ne(r).baseUrl,o=n(`${r.toUpperCase().replace(/-/g,"_")}_EMBEDDING_MODEL`)}let a=m(n("TOOLNET_EMBEDDING_ACCOUNT_ID"),i),c=m(n("TOOLNET_EMBEDDING_BASE_URL"),s,e.defaultBaseUrl),l=r==="cloudflare"&&!c&&a?`https://api.cloudflare.com/client/v4/accounts/${a}/ai/v1`:c;return{provider:r,apiKey:m(n("TOOLNET_EMBEDDING_API_KEY"),t),baseUrl:l,model:m(n("TOOLNET_EMBEDDING_MODEL"),o),accountId:a}}function we(){let r=Rt(),e=It();return{llm:r,embedding:e,legacy:{llm:!n("TOOLNET_LLM_PROVIDER")&&!!xe(),embedding:!n("TOOLNET_EMBEDDING_PROVIDER")&&!!(n("HF_TOKEN")||n("HF_EMBEDDING_MODEL"))}}}var J=class{constructor(e){this.options=e}options;name="gemini";async embed(e){let t=this.options.model.replace(/^models\//,""),o=`${this.options.baseUrl?.replace(/\/+$/,"")||"https://generativelanguage.googleapis.com/v1beta"}/models/${encodeURIComponent(t)}:embedContent?key=${encodeURIComponent(this.options.apiKey)}`,i=await fetch(o,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({model:`models/${t}`,content:{parts:[{text:e}]}})});if(!i.ok)throw new Error(`Gemini embedding failed: ${i.status} ${await i.text()}`);let c=(await i.json()).embedding?.values;if(!Array.isArray(c)||c.length===0)throw new Error("Gemini embedding returned invalid response");return c}async embedMany(e){let t=[];for(let s of e)t.push(await this.embed(s));return t}};import{createHash as Tt}from"node:crypto";var Q=class{name="hash-fallback";dimensions=128;async embed(e){let t=new Array(this.dimensions).fill(0),s=e.toLowerCase().split(/\s+/).filter(Boolean);for(let o of s){let a=Tt("sha256").update(o).digest().readUInt32BE(0)%this.dimensions;t[a]+=1}return _t(t)}async embedMany(e){return Promise.all(e.map(t=>this.embed(t)))}};function _t(r){let e=Math.sqrt(r.reduce((t,s)=>t+s*s,0));return e===0?r:r.map(t=>t/e)}var Z=class{constructor(e){this.options=e;this.dimensions=e.dimensions}options;name="openai-compatible";dimensions;async embed(e){let[t]=await this.embedMany([e]);return t}async embedMany(e){if(e.length===0)return[];let t=this.options.baseUrl.replace(/\/+$/,""),s=await fetch(`${t}/embeddings`,{method:"POST",headers:{"content-type":"application/json",...this.options.apiKey?{authorization:`Bearer ${this.options.apiKey}`}:{}},body:JSON.stringify({model:this.options.model,input:e})});if(!s.ok)throw new Error(`Embedding failed: ${s.status} ${await s.text()}`);let i=(await s.json()).data?.sort((a,c)=>(a.index??0)-(c.index??0)).map(a=>a.embedding);if(!i||i.length!==e.length||i.some(a=>!Array.isArray(a)))throw new Error("Embedding provider returned invalid response");return i}};var X=class{constructor(e){this.options=e}options;name="huggingface";dimensions=384;async embed(e){let[t]=await this.embedMany([e]);return t}async embedMany(e){if(e.length===0)return[];let t=`https://router.huggingface.co/hf-inference/models/${this.options.model}/pipeline/feature-extraction`,s=await fetch(t,{method:"POST",headers:{Authorization:`Bearer ${this.options.token}`,"Content-Type":"application/json"},body:JSON.stringify({inputs:e,options:{wait_for_model:!0}})});if(!s.ok)throw new Error(`HF embedding failed: ${s.status} ${await s.text()}`);let o=await s.json();if(!Array.isArray(o))throw new Error("HF embedding returned invalid response");return o.map(i=>Mt(i))}};function Mt(r){if(!Array.isArray(r))throw new Error("Invalid embedding");if(r.length>0&&typeof r[0]=="number")return r;if(r.length>0&&Array.isArray(r[0])){let e=r;if(e.length===0)return[];let t=e[0].length,s=new Array(t).fill(0);for(let o of e)for(let i=0;i<t;i++)s[i]+=o[i]??0;for(let o=0;o<t;o++)s[o]/=e.length;return s}throw new Error("Unsupported embedding shape")}var ie=class{constructor(e,t=new Q){this.primary=e;this.fallback=t}primary;fallback;name="resilient";async embed(e){if(this.primary)try{return await this.primary.embed(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embed(e)}async embedMany(e){if(this.primary)try{return await this.primary.embedMany(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embedMany(e)}};function Nt(r){return`https://api.cloudflare.com/client/v4/accounts/${r}/ai/v1`}function Ot(){let r=we().embedding;if(r.provider==="local")return null;if(r.provider==="huggingface")return!r.apiKey||!r.model?null:new X({token:r.apiKey,model:r.model});if(r.provider==="gemini")return!r.apiKey||!r.model?null:new J({apiKey:r.apiKey,model:r.model,baseUrl:r.baseUrl});let e=r.baseUrl;return r.provider==="cloudflare"&&!e&&r.accountId&&(e=Nt(r.accountId)),e&&r.model?new Z({apiKey:r.apiKey,baseUrl:e,model:r.model}):null}function Pe(){return new ie(Ot())}async function jt(){let r=le(),e=new F().detect(),t=ye({provider:r.storage.provider,huggingface:r.storage.huggingface,localRoot:r.storage.localRoot}),s=new z(t,e.id,e.name,e.remote??e.name),o=new A,i=await new j(s).load(e.id);if(!i)throw new Error("Code graph missing. Run npm run code:index:test first.");o.import(i.symbols,i.edges);let a=new W({projectId:e.id,rootPath:e.rootPath,model:process.env.HF_EMBEDDING_MODEL??"sentence-transformers/all-MiniLM-L6-v2",storage:s,embeddings:Pe(),graph:o}),c=await a.initialize(),l=process.argv.slice(2).join(" ")||"memory storage hugging face",d=await a.search(l,5);console.log({ok:!0,query:l,...c,results:d.map(p=>({score:Number(p.score.toFixed(4)),file:p.chunk.filePath,symbol:p.chunk.symbolName,lines:`${p.chunk.startLine}-${p.chunk.endLine}`,preview:p.chunk.content.replace(/\s+/g," ").slice(0,140)}))})}jt().catch(r=>{console.error(r),process.exit(1)});
|
|
1
|
+
import"dotenv/config";import{existsSync as Se,readFileSync as be}from"node:fs";import{homedir as xe}from"node:os";import{join as Ee}from"node:path";function Ce(s){let e=s.trim();return e.length>=2&&e.startsWith('"')&&e.endsWith('"')?(e=e.slice(1,-1),e.replace(/\\n/g,`
|
|
2
|
+
`).replace(/\\r/g,"\r").replace(/\\t/g," ").replace(/\\"/g,'"').replace(/\\\\/g,"\\")):e.length>=2&&e.startsWith("'")&&e.endsWith("'")?e.slice(1,-1):e}function we(){let s=process.env.TOOLNET_GLOBAL_ENV??Ee(xe(),".config","toolnet-memory",".env");if(!Se(s))return;let e=be(s,"utf8");for(let t of e.split(/\r?\n/)){let r=t.trim();if(!r||r.startsWith("#"))continue;r.startsWith("export ")&&(r=r.slice(7));let o=r.indexOf("=");if(o<=0)continue;let n=r.slice(0,o).trim();/^[A-Za-z_][A-Za-z0-9_]*$/.test(n)&&process.env[n]===void 0&&(process.env[n]=Ce(r.slice(o+1)))}}we();function R(s,e){return s===void 0?e:["1","true","yes","on"].includes(s.toLowerCase())}function j(s,e){if(!s)return e;let t=Number(s);return Number.isFinite(t)?t:e}function re(){return{memory:{autoCapture:R(process.env.MEMORY_AUTO_CAPTURE,!0),autoRetrieve:R(process.env.MEMORY_AUTO_RETRIEVE,!0),autoSummarize:R(process.env.MEMORY_AUTO_SUMMARIZE,!0),autoSync:R(process.env.MEMORY_AUTO_SYNC,!0)},retrieval:{maxCandidates:j(process.env.MEMORY_MAX_CANDIDATES,50),rerankTop:j(process.env.MEMORY_RERANK_TOP,10),finalContext:j(process.env.MEMORY_FINAL_CONTEXT,5),tokenBudget:j(process.env.MEMORY_TOKEN_BUDGET,2e3)},storage:{provider:process.env.MEMORY_STORAGE_PROVIDER??"huggingface",r2:{accountId:process.env.R2_ACCOUNT_ID,bucket:process.env.R2_BUCKET,accessKeyId:process.env.R2_ACCESS_KEY_ID,secretAccessKey:process.env.R2_SECRET_ACCESS_KEY},s3:{endpoint:process.env.S3_ENDPOINT,region:process.env.S3_REGION,bucket:process.env.S3_BUCKET,accessKeyId:process.env.S3_ACCESS_KEY_ID,secretAccessKey:process.env.S3_SECRET_ACCESS_KEY,forcePathStyle:R(process.env.S3_FORCE_PATH_STYLE,!1)},huggingface:{namespace:process.env.HF_NAMESPACE,bucket:process.env.HF_BUCKET,accessKeyId:process.env.HF_S3_ACCESS_KEY_ID,secretAccessKey:process.env.HF_S3_SECRET_ACCESS_KEY},localRoot:process.env.MEMORY_LOCAL_STORAGE_PATH},cache:{maxMb:j(process.env.MEMORY_LOCAL_CACHE_MB,200)}}}import{createHash as ve}from"node:crypto";import{existsSync as q,mkdirSync as Pe,readFileSync as ke,renameSync as Te,writeFileSync as Re}from"node:fs";import{basename as je,dirname as O,join as F,parse as ne,resolve as A}from"node:path";var ie=".toolnet",Ae="project.json";function Le(s){return ve("sha256").update(s).digest("hex").slice(0,16)}function J(s){return F(s,ie,Ae)}function Ne(s){return q(J(s))}function Me(s,e){let t=A(s),r=ne(t).root;for(;;){if(Ne(t))return t;if(t===r||e&&t===A(e))break;let o=O(t);if(o===t)break;t=o}return null}function Ie(s){let e=A(s),t=ne(e).root,r=[".git","package.json","pyproject.toml","Cargo.toml","go.mod","composer.json"];for(;;){if(r.some(n=>q(F(e,n))))return e;if(e===t)break;let o=O(e);if(o===e)break;e=o}return A(s)}function Oe(s){let e;try{e=JSON.parse(ke(s,"utf8"))}catch(o){throw new Error(`Invalid ToolNet project manifest: ${s}: ${o instanceof Error?o.message:String(o)}`)}if(!e||typeof e!="object")throw new Error(`Invalid ToolNet project manifest: ${s}`);let t=e;if(typeof t.id!="string"||!t.id.trim())throw new Error(`ToolNet project manifest is missing id: ${s}`);if(typeof t.name!="string"||!t.name.trim())throw new Error(`ToolNet project manifest is missing name: ${s}`);let r=new Date().toISOString();return{version:1,id:t.id,name:t.name,remote:typeof t.remote=="string"&&t.remote.trim()?t.remote:t.name,rootPath:typeof t.rootPath=="string"?t.rootPath:O(O(s)),createdAt:typeof t.createdAt=="string"?t.createdAt:r,updatedAt:typeof t.updatedAt=="string"?t.updatedAt:r,graphVersion:typeof t.graphVersion=="number"?t.graphVersion:0,memoryVersion:typeof t.memoryVersion=="number"?t.memoryVersion:0,metadata:t.metadata&&typeof t.metadata=="object"?t.metadata:void 0}}function se(s,e){let t=F(s,ie);Pe(t,{recursive:!0});let r=J(s),o=`${r}.tmp-${process.pid}`;Re(o,JSON.stringify(e,null,2)+`
|
|
3
|
+
`,{encoding:"utf8",mode:384}),Te(o,r)}function oe(s,e){return{id:s.id,name:s.name,remote:s.remote,rootPath:e,createdAt:s.createdAt,updatedAt:s.updatedAt,graphVersion:s.graphVersion,memoryVersion:s.memoryVersion,metadata:s.metadata}}var _=class{detect(e=process.cwd()){let t=A(e),r=Ie(t),n=[".git","package.json","pyproject.toml","Cargo.toml","go.mod","composer.json"].some(d=>q(F(r,d))),i=Me(t,n?r:void 0);if(i){let d=J(i),m=Oe(d);return m.rootPath!==i&&(m.rootPath=i,m.updatedAt=new Date().toISOString(),se(i,m)),oe(m,i)}let c=new Date().toISOString(),l=je(r),a={version:1,id:Le(r),name:l,remote:l,rootPath:r,createdAt:c,updatedAt:c,graphVersion:0,memoryVersion:0};return se(r,a),oe(a,r)}};var _e=[{type:"openai_key",regex:/\bsk-[A-Za-z0-9_\-]{20,}\b/g},{type:"huggingface_token",regex:/\bhf_[A-Za-z0-9]{20,}\b/g},{type:"hf_s3_access_key",regex:/\bHFAK[A-Za-z0-9]{8,}\b/g},{type:"bearer_token",regex:/\bBearer\s+[A-Za-z0-9._\-]{16,}\b/gi},{type:"jwt",regex:/\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\b/g},{type:"private_key",regex:/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g},{type:"password_assignment",regex:/\b(password|passwd|pwd)\s*[:=]\s*["']?[^"'\s]{6,}["']?/gi},{type:"secret_assignment",regex:/\b(secret|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*["']?[^"'\s]{8,}["']?/gi},{type:"cookie",regex:/\b(cookie|set-cookie)\s*[:=]\s*[^;\n]{8,}/gi}],$=class{scan(e){let t=[];for(let r of _e){let o=new RegExp(r.regex.source,r.regex.flags);for(let n of e.matchAll(o))t.push({type:r.type,value:n[0]})}return t}hasSecrets(e){return this.scan(e).length>0}};var L=class{scanner=new $;sanitize(e){let t=e,r=this.scanner.scan(e),o=new Set;for(let n of r)o.add(n.type),t=t.split(n.value).join(`[REDACTED:${n.type}]`);return{text:t,redacted:r.length,secretTypes:[...o]}}sanitizeValue(e){if(typeof e=="string")return this.sanitize(e).text;if(Array.isArray(e))return e.map(t=>this.sanitizeValue(t));if(e&&typeof e=="object"){let t={};for(let[r,o]of Object.entries(e)){let n=r.toLowerCase();n.includes("password")||n.includes("secret")||n.includes("token")||n.includes("cookie")||n.includes("authorization")?t[r]="[REDACTED]":t[r]=this.sanitizeValue(o)}return t}return e}};import{homedir as nt}from"node:os";import{join as it}from"node:path";import{DeleteObjectCommand as Fe,GetObjectCommand as $e,HeadObjectCommand as De,ListObjectsV2Command as Ge,PutObjectCommand as Ke,S3Client as ze}from"@aws-sdk/client-s3";import{getSignedUrl as Be}from"@aws-sdk/s3-request-presigner";var D=class{name="huggingface";client;bucket;constructor(e){this.bucket=e.bucket,this.client=new ze({region:"us-east-1",endpoint:`https://s3.hf.co/${e.namespace}`,forcePathStyle:!0,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new Ke({Bucket:this.bucket,Key:e,Body:o,ContentType:r}))}async get(e){let t=await Be(this.client,new $e({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`HF download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new De({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new Fe({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let o=await this.client.send(new Ge({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of o.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=o.IsTruncated?o.NextContinuationToken:void 0}while(r);return t}};import{access as ae,mkdir as He,readFile as We,readdir as Ue,rm as Ve,stat as ce,writeFile as Ye}from"node:fs/promises";import{dirname as qe,join as Je,relative as le,resolve as Qe}from"node:path";var N=class{constructor(e){this.root=e}root;name="local";path(e){let t=e.replace(/^\/+/,"");return Qe(this.root,t)}async put(e,t){let r=this.path(e);await He(qe(r),{recursive:!0}),await Ye(r,t)}async get(e){try{return await We(this.path(e))}catch(t){if(typeof t=="object"&&t!==null&&"code"in t&&t.code==="ENOENT")return null;throw t}}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await ae(this.path(e)),!0}catch{return!1}}async delete(e){await Ve(this.path(e),{force:!0})}async list(e=""){let t=this.path(e),r=[];try{await ae(t)}catch{return r}let o=async i=>{let c=await Ue(i,{withFileTypes:!0});for(let l of c){let a=Je(i,l.name);if(l.isDirectory()){await o(a);continue}let d=await ce(a);r.push({key:le(this.root,a),size:d.size,updatedAt:d.mtime.toISOString()})}},n=await ce(t);return n.isDirectory()?await o(t):r.push({key:le(this.root,t),size:n.size,updatedAt:n.mtime.toISOString()}),r}};import{DeleteObjectCommand as Ze,GetObjectCommand as Xe,HeadObjectCommand as et,ListObjectsV2Command as tt,PutObjectCommand as rt,S3Client as st}from"@aws-sdk/client-s3";import{getSignedUrl as ot}from"@aws-sdk/s3-request-presigner";var M=class{name;client;bucket;constructor(e){this.name=e.name??"s3",this.bucket=e.bucket,this.client=new st({region:e.region??"us-east-1",endpoint:e.endpoint||void 0,forcePathStyle:e.forcePathStyle??!1,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new rt({Bucket:this.bucket,Key:e,Body:o,ContentType:r}))}async get(e){let t=await ot(this.client,new Xe({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`${this.name} download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new et({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new Ze({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let o=await this.client.send(new tt({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of o.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=o.IsTruncated?o.NextContinuationToken:void 0}while(r);return t}};function Q(s,e){return console.warn(e),new N(s)}function de(s){let e=s.localRoot??it(nt(),".toolnet-memory","storage");if(s.provider==="r2"){let t=s.r2;return t?.accountId&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new M({name:"r2",endpoint:`https://${t.accountId}.r2.cloudflarestorage.com`,region:"auto",bucket:t.bucket,forcePathStyle:!0,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):Q(e,"[storage] Cloudflare R2 credentials missing. Using local fallback.")}if(s.provider==="s3"){let t=s.s3;return t?.bucket&&t.accessKeyId&&t.secretAccessKey?new M({name:"s3",endpoint:t.endpoint,region:t.region??"us-east-1",bucket:t.bucket,forcePathStyle:t.forcePathStyle??!1,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):Q(e,"[storage] S3 credentials missing. Using local fallback.")}if(s.provider==="huggingface"){let t=s.huggingface;return t?.namespace&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new D({namespace:t.namespace,bucket:t.bucket,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):Q(e,"[storage] Hugging Face credentials missing. Using local fallback.")}return new N(e)}var I=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"graph","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var G=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","chunks","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var K=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","vectors","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};function pe(s){let e=s.trim().replace(/\s+/g,"_").replace(/[^A-Za-z0-9._-]/g,"_").replace(/_+/g,"_").replace(/^\.+|\.+$/g,"").slice(0,100);if(!e||e==="."||e==="..")throw new Error("Invalid project storage folder");return e}function me(s){let e=s.replace(/^\/+/,"");if(e.startsWith("memories/"))return"memory/records/"+e.slice(9);if(e.startsWith("vectors/"))return"memory/vectors/"+e.slice(8);if(e.startsWith("graph/"))return"code/graph/"+e.slice(6);let t=e.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);if(t)return`${t[1]}memory/records/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/),t)return`${t[1]}memory/vectors/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/),t)return`${t[1]}code/graph/${t[2]}`;let r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);if(r)return`${r[1]}memory/records/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/),r)return`${r[1]}memory/vectors/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/),r)return`${r[1]}code/graph/${r[2]}`;let o=e.match(/^(projects\/[^/]+\/)memories\/(.+)$/);return o?`${o[1]}memory/records/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)vectors\/(.+)$/),o?`${o[1]}memory/vectors/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)graph\/(.+)$/),o?`${o[1]}code/graph/${o[2]}`:e))}var z=class{constructor(e,t,r,o){this.provider=e;this.name=e.name,this.projectId=t,this.projectName=r,this.folder=pe(o??r),this.sourcePrefix=`projects/${t}`,this.targetPrefix=`projects/${this.folder}`}provider;name;folder;sourcePrefix;targetPrefix;projectId;projectName;registryPromise;async registerProject(){let e=`${this.targetPrefix}/project.json`,t=new Date().toISOString(),r=t,o=await this.provider.getText(e);if(o){let i;try{i=JSON.parse(o)}catch(c){throw new Error(`Invalid remote ToolNet project manifest at ${e}: ${c instanceof Error?c.message:String(c)}`)}if(typeof i.id=="string"&&i.id!==this.projectId)throw new Error(["ToolNet remote project namespace collision.",`Remote folder: ${this.targetPrefix}`,`Existing project id: ${i.id}`,`Current project id: ${this.projectId}`,"Refusing to mix data from two projects."].join(" "));typeof i.createdAt=="string"&&(r=i.createdAt)}let n={version:1,id:this.projectId,name:this.projectName,remote:this.folder,createdAt:r,updatedAt:t};await this.provider.put(e,JSON.stringify(n,null,2)+`
|
|
4
|
+
`,"application/json")}async ensureRegistered(){return this.registryPromise||(this.registryPromise=this.registerProject()),this.registryPromise}key(e){if(e=me(e),e===this.sourcePrefix)return this.targetPrefix;if(e.startsWith(`${this.sourcePrefix}/`))return this.targetPrefix+e.slice(this.sourcePrefix.length);if(e===this.targetPrefix||e.startsWith(`${this.targetPrefix}/`))return e;if(e.startsWith("projects/"))throw new Error(["Cross-project storage access denied.",`Current project: ${this.targetPrefix}`,`Requested key: ${e}`].join(" "));return e}async put(e,t,r){return await this.ensureRegistered(),this.provider.put(this.key(e),t,r)}async get(e){return await this.ensureRegistered(),this.provider.get(this.key(e))}async getText(e){return await this.ensureRegistered(),this.provider.getText(this.key(e))}async delete(e){return await this.ensureRegistered(),this.provider.delete(this.key(e))}async exists(e){return await this.ensureRegistered(),this.provider.exists(this.key(e))}async list(e){return await this.ensureRegistered(),this.provider.list(this.key(e))}};import*as u from"typescript";var v=class{symbols=new Map;edges=new Map;addSymbol(e){this.symbols.set(e.id,e)}addEdge(e){this.edges.set(e.id,e)}getSymbol(e){return this.symbols.get(e)}findByName(e,t){return[...this.symbols.values()].filter(r=>r.projectId===e&&(r.name===t||r.qualifiedName===t))}allSymbols(e){return[...this.symbols.values()].filter(t=>t.projectId===e)}allEdges(e){return[...this.edges.values()].filter(t=>t.projectId===e)}import(e,t){for(let r of e)this.addSymbol(r);for(let r of t)this.addEdge(r)}clearProject(e){for(let[t,r]of this.symbols)r.projectId===e&&this.symbols.delete(t);for(let[t,r]of this.edges)r.projectId===e&&this.edges.delete(t)}};import{execFile as dt}from"node:child_process";import{promisify as pt}from"node:util";var js=pt(dt);import{createHash as mt}from"node:crypto";import{readFile as ft}from"node:fs/promises";import{join as ut}from"node:path";function fe(s){return mt("sha256").update(s).digest("hex")}var B=class{sanitizer=new L;async build(e,t,r,o={}){let n=o.maxLines??80,i=o.overlapLines??10,c=new Map;for(let a of r){let d=c.get(a.filePath)??[];d.push(a),c.set(a.filePath,d)}let l=[];for(let[a,d]of c){let m;try{m=await ft(ut(t,a),"utf8")}catch{continue}let g=m.split(/\r?\n/),S=d.filter(h=>h.type!=="file"&&typeof h.startLine=="number"&&typeof h.endLine=="number");if(S.length>0)for(let h of S)l.push(...this.chunkRange({projectId:e,filePath:a,lines:g,symbol:h,startLine:h.startLine,endLine:h.endLine,maxLines:n,overlapLines:i}));else l.push(...this.chunkRange({projectId:e,filePath:a,lines:g,startLine:1,endLine:Math.max(1,g.length),maxLines:n,overlapLines:i}))}return l}chunkRange(e){let{projectId:t,filePath:r,lines:o,symbol:n,maxLines:i}=e,c=Math.min(e.overlapLines,Math.max(0,i-1)),l=Math.max(1,e.startLine),a=Math.min(o.length,Math.max(l,e.endLine)),d=[],m=l,g=0;for(;m<=a;){let S=Math.min(a,m+i-1),h=o.slice(m-1,S).join(`
|
|
5
|
+
`).trim();if(h){let x=this.sanitizer.sanitize(h).text,P=fe(x),k=[t,r,n?.id??"file",String(g),P].join(":");d.push({id:fe(k).slice(0,32),projectId:t,filePath:r,symbolId:n?.id,symbolName:n?.qualifiedName??n?.name,symbolType:n?.type,startLine:m,endLine:S,content:x,contentHash:P})}if(S>=a)break;m=S-c+1,g++}return d}};var H=class{records=new Map;upsert(e){this.records.set(e.id,e)}get(e){return this.records.get(e)}has(e){return this.records.has(e)}remove(e){return this.records.delete(e)}clearProject(e){let t=0;for(let[r,o]of this.records)o.projectId===e&&(this.records.delete(r),t++);return t}exportProject(e){return[...this.records.values()].filter(t=>t.projectId===e)}importRecords(e){let t=0;for(let r of e)!r?.id||!r?.projectId||!Array.isArray(r.vector)||(this.records.set(r.id,r),t++);return t}search(e,t,r=10){let o=[];for(let n of this.records.values()){if(n.projectId!==e)continue;let i=gt(t,n.vector);o.push({id:n.id,score:i,metadata:n.metadata})}return o.sort((n,i)=>i.score-n.score).slice(0,r)}size(){return this.records.size}};function gt(s,e){if(s.length===0||e.length===0||s.length!==e.length)return 0;let t=0,r=0,o=0;for(let n=0;n<s.length;n++){let i=s[n]??0,c=e[n]??0;t+=i*c,r+=i*i,o+=c*c}return r===0||o===0?0:t/(Math.sqrt(r)*Math.sqrt(o))}function Z(s){return s.toLowerCase().normalize("NFKC").replace(/[^\p{L}\p{N}_\-./]+/gu," ").split(/\s+/).map(e=>e.trim()).filter(e=>e.length>1)}function ue(s,e){let t=s.toLowerCase().replaceAll("\\","/"),r=e.toLowerCase(),o=/\b(test|tests|spec|fixture|mock)\b/.test(r);return(t.startsWith("tests/")||t.includes("/tests/")||t.includes("__tests__")||/\.(test|spec)\.[^.]+$/.test(t))&&!o?.35:t.includes("/generated/")||t.includes(".generated.")?.25:1}var W=class{constructor(e){this.options=e}options;vectors=new H;chunks=new Map;async initialize(){let{projectId:e,rootPath:t,storage:r,embeddings:o,graph:n,model:i}=this.options,c=new G(r),l=new K(r),a=await new B().build(e,t,n.allSymbols(e));this.chunks.clear();for(let f of a)this.chunks.set(f.id,f);let d=await c.load(e),m=!d||d.chunks.length!==a.length||d.chunks.some(f=>!this.chunks.has(f.id));m&&await c.save({version:1,projectId:e,updatedAt:new Date().toISOString(),chunks:a});let g=await l.load(e),S=0;g&&g.model===i&&(S=this.vectors.importRecords(g.records));let h=new Set(a.map(f=>f.id)),x=0;for(let f of this.vectors.exportProject(e))h.has(f.id)||this.vectors.remove(f.id)&&x++;let P=a.filter(f=>{let C=this.vectors.get(f.id);return C?C.metadata?.contentHash!==f.contentHash:!0}),k=0,ee=8,T=this.vectors.exportProject(e)[0]?.vector.length;for(let f=0;f<P.length;f+=ee){let C=P.slice(f,f+ee),he=C.map(b=>[`file: ${b.filePath}`,b.symbolName?`symbol: ${b.symbolName}`:"",b.symbolType?`type: ${b.symbolType}`:"",b.content].filter(Boolean).join(`
|
|
6
|
+
`)),ye=await o.embedMany(he);for(let b=0;b<C.length;b++){let E=C[b],w=ye[b];if(!(!E||!w||w.length===0)){if(T===void 0&&(T=w.length),w.length!==T){console.warn(`[code-semantic] skip dimension mismatch: ${w.length} != ${T}`);continue}this.vectors.upsert({id:E.id,projectId:e,vector:w,metadata:{contentHash:E.contentHash,filePath:E.filePath,symbolName:E.symbolName,symbolType:E.symbolType,startLine:E.startLine,endLine:E.endLine}}),k++}}}let Y=this.vectors.exportProject(e),te=k>0||x>0||!g||g.model!==i;return te&&await l.save({version:1,projectId:e,model:i,dimensions:Y[0]?.vector.length??T??0,updatedAt:new Date().toISOString(),records:Y}),{chunks:a.length,vectorsLoaded:S,vectorsIndexed:k,vectorsRemoved:x,totalVectors:Y.length,savedChunks:m,savedVectors:te}}async search(e,t=8){let r=await this.options.embeddings.embed(e),o=this.vectors.search(this.options.projectId,r,Math.max(t*4,20)),n=new Set(Z(e)),i=[];for(let c of o){let l=this.chunks.get(c.id);if(!l)continue;let a=new Set(Z([l.filePath,l.symbolName??"",l.symbolType??"",l.content].join(" "))),d=0;for(let x of n)a.has(x)&&d++;let m=n.size===0?0:d/n.size,g=Math.max(0,c.score),S=ue(l.filePath,e),h=(g*.75+m*.25)*S;i.push({chunk:l,score:h,vectorScore:g,lexicalScore:m})}return i.sort((c,l)=>l.score-c.score).slice(0,t)}size(){return this.chunks.size}};import*as y from"typescript";import*as p from"typescript";var _o=new Set([p.SyntaxKind.EqualsToken,p.SyntaxKind.PlusEqualsToken,p.SyntaxKind.MinusEqualsToken,p.SyntaxKind.AsteriskEqualsToken,p.SyntaxKind.SlashEqualsToken,p.SyntaxKind.PercentEqualsToken,p.SyntaxKind.QuestionQuestionEqualsToken,p.SyntaxKind.AmpersandAmpersandEqualsToken,p.SyntaxKind.BarBarEqualsToken]);var U=class{constructor(e){this.options=e}options;name="huggingface";dimensions=384;async embed(e){let[t]=await this.embedMany([e]);return t}async embedMany(e){if(e.length===0)return[];let t=`https://router.huggingface.co/hf-inference/models/${this.options.model}/pipeline/feature-extraction`,r=await fetch(t,{method:"POST",headers:{Authorization:`Bearer ${this.options.token}`,"Content-Type":"application/json"},body:JSON.stringify({inputs:e,options:{wait_for_model:!0}})});if(!r.ok)throw new Error(`HF embedding failed: ${r.status} ${await r.text()}`);let o=await r.json();if(!Array.isArray(o))throw new Error("HF embedding returned invalid response");return o.map(n=>yt(n))}};function yt(s){if(!Array.isArray(s))throw new Error("Invalid embedding");if(s.length>0&&typeof s[0]=="number")return s;if(s.length>0&&Array.isArray(s[0])){let e=s;if(e.length===0)return[];let t=e[0].length,r=new Array(t).fill(0);for(let o of e)for(let n=0;n<t;n++)r[n]+=o[n]??0;for(let o=0;o<t;o++)r[o]/=e.length;return r}throw new Error("Unsupported embedding shape")}import{createHash as St}from"node:crypto";var V=class{name="hash-fallback";dimensions=128;async embed(e){let t=new Array(this.dimensions).fill(0),r=e.toLowerCase().split(/\s+/).filter(Boolean);for(let o of r){let i=St("sha256").update(o).digest().readUInt32BE(0)%this.dimensions;t[i]+=1}return bt(t)}async embedMany(e){return Promise.all(e.map(t=>this.embed(t)))}};function bt(s){let e=Math.sqrt(s.reduce((t,r)=>t+r*r,0));return e===0?s:s.map(t=>t/e)}var X=class{constructor(e,t=new V){this.primary=e;this.fallback=t}primary;fallback;name="resilient";async embed(e){if(this.primary)try{return await this.primary.embed(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embed(e)}async embedMany(e){if(this.primary)try{return await this.primary.embedMany(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embedMany(e)}};function ge(){let s=process.env.HF_TOKEN,e=process.env.HF_EMBEDDING_MODEL??"sentence-transformers/all-MiniLM-L6-v2",t=s?new U({token:s,model:e}):null;return new X(t)}async function xt(){let s=re(),e=new _().detect(),t=de({provider:s.storage.provider,huggingface:s.storage.huggingface,localRoot:s.storage.localRoot}),r=new z(t,e.id,e.name,e.remote??e.name),o=new v,n=await new I(r).load(e.id);if(!n)throw new Error("Code graph missing. Run npm run code:index:test first.");o.import(n.symbols,n.edges);let i=new W({projectId:e.id,rootPath:e.rootPath,model:process.env.HF_EMBEDDING_MODEL??"sentence-transformers/all-MiniLM-L6-v2",storage:r,embeddings:ge(),graph:o}),c=await i.initialize(),l=process.argv.slice(2).join(" ")||"memory storage hugging face",a=await i.search(l,5);console.log({ok:!0,query:l,...c,results:a.map(d=>({score:Number(d.score.toFixed(4)),file:d.chunk.filePath,symbol:d.chunk.symbolName,lines:`${d.chunk.startLine}-${d.chunk.endLine}`,preview:d.chunk.content.replace(/\s+/g," ").slice(0,140)}))})}xt().catch(s=>{console.error(s),process.exit(1)});
|