toolnet-memory 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/agy-hook.js +101 -42
- package/bundle/agy.js +114 -38
- package/bundle/api.js +52 -0
- package/bundle/auto-integrate.js +1005 -123
- package/bundle/claude-hook.js +144 -0
- package/bundle/claude.js +3 -0
- package/bundle/codex-context.js +54 -44
- package/bundle/codex.js +85 -47
- package/bundle/context-runtime.js +126 -31
- package/bundle/context.js +15 -14
- package/bundle/continuity-certify.js +50 -0
- package/bundle/doctor.js +6 -3
- package/bundle/full-index.js +24 -8
- package/bundle/graph-index.js +4 -4
- package/bundle/handoff-refresh.js +12 -0
- package/bundle/impact.js +3 -3
- package/bundle/incremental.js +4 -4
- package/bundle/init.js +1069 -0
- package/bundle/learner.js +5 -5
- package/bundle/mcp.js +56 -7
- package/bundle/memory-agent.js +48 -0
- package/bundle/memory-query.js +7 -0
- package/bundle/opencode.js +1028 -158
- package/bundle/production-certify.js +5 -0
- package/bundle/project-manual.js +6 -6
- package/bundle/provider.js +6 -0
- package/bundle/recovery-certify.js +56 -0
- package/bundle/runtime.js +11 -11
- package/bundle/semantic.js +6 -6
- package/bundle/service-cli.js +3 -0
- package/bundle/service.js +6 -0
- package/bundle/setup.js +1013 -178
- package/bundle/snapshot.js +4 -4
- package/bundle/update.js +4 -1
- package/bundle/work-continuity.js +7 -7
- package/package.json +1 -1
package/bundle/semantic.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import"dotenv/config";import{existsSync as
|
|
2
|
-
`).replace(/\\r/g,"\r").replace(/\\t/g," ").replace(/\\"/g,'"').replace(/\\\\/g,"\\")):e.length>=2&&e.startsWith("'")&&e.endsWith("'")?e.slice(1,-1):e}function
|
|
3
|
-
`,{encoding:"utf8",mode:384}),Te(o,r)}function oe(s,e){return{id:s.id,name:s.name,remote:s.remote,rootPath:e,createdAt:s.createdAt,updatedAt:s.updatedAt,graphVersion:s.graphVersion,memoryVersion:s.memoryVersion,metadata:s.metadata}}var _=class{detect(e=process.cwd()){let t=A(e),r=Ie(t),n=[".git","package.json","pyproject.toml","Cargo.toml","go.mod","composer.json"].some(d=>q(F(r,d))),i=Me(t,n?r:void 0);if(i){let d=J(i),m=Oe(d);return m.rootPath!==i&&(m.rootPath=i,m.updatedAt=new Date().toISOString(),se(i,m)),oe(m,i)}let c=new Date().toISOString(),l=je(r),a={version:1,id:Le(r),name:l,remote:l,rootPath:r,createdAt:c,updatedAt:c,graphVersion:0,memoryVersion:0};return se(r,a),oe(a,r)}};var _e=[{type:"openai_key",regex:/\bsk-[A-Za-z0-9_\-]{20,}\b/g},{type:"huggingface_token",regex:/\bhf_[A-Za-z0-9]{20,}\b/g},{type:"hf_s3_access_key",regex:/\bHFAK[A-Za-z0-9]{8,}\b/g},{type:"bearer_token",regex:/\bBearer\s+[A-Za-z0-9._\-]{16,}\b/gi},{type:"jwt",regex:/\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\b/g},{type:"private_key",regex:/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g},{type:"password_assignment",regex:/\b(password|passwd|pwd)\s*[:=]\s*["']?[^"'\s]{6,}["']?/gi},{type:"secret_assignment",regex:/\b(secret|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*["']?[^"'\s]{8,}["']?/gi},{type:"cookie",regex:/\b(cookie|set-cookie)\s*[:=]\s*[^;\n]{8,}/gi}],$=class{scan(e){let t=[];for(let r of _e){let o=new RegExp(r.regex.source,r.regex.flags);for(let n of e.matchAll(o))t.push({type:r.type,value:n[0]})}return t}hasSecrets(e){return this.scan(e).length>0}};var L=class{scanner=new $;sanitize(e){let t=e,r=this.scanner.scan(e),o=new Set;for(let n of r)o.add(n.type),t=t.split(n.value).join(`[REDACTED:${n.type}]`);return{text:t,redacted:r.length,secretTypes:[...o]}}sanitizeValue(e){if(typeof e=="string")return this.sanitize(e).text;if(Array.isArray(e))return e.map(t=>this.sanitizeValue(t));if(e&&typeof e=="object"){let t={};for(let[r,o]of Object.entries(e)){let n=r.toLowerCase();n.includes("password")||n.includes("secret")||n.includes("token")||n.includes("cookie")||n.includes("authorization")?t[r]="[REDACTED]":t[r]=this.sanitizeValue(o)}return t}return e}};import{homedir as nt}from"node:os";import{join as it}from"node:path";import{DeleteObjectCommand as Fe,GetObjectCommand as $e,HeadObjectCommand as De,ListObjectsV2Command as Ge,PutObjectCommand as Ke,S3Client as ze}from"@aws-sdk/client-s3";import{getSignedUrl as Be}from"@aws-sdk/s3-request-presigner";var D=class{name="huggingface";client;bucket;constructor(e){this.bucket=e.bucket,this.client=new ze({region:"us-east-1",endpoint:`https://s3.hf.co/${e.namespace}`,forcePathStyle:!0,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new Ke({Bucket:this.bucket,Key:e,Body:o,ContentType:r}))}async get(e){let t=await Be(this.client,new $e({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`HF download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new De({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new Fe({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let o=await this.client.send(new Ge({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of o.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=o.IsTruncated?o.NextContinuationToken:void 0}while(r);return t}};import{access as ae,mkdir as He,readFile as We,readdir as Ue,rm as Ve,stat as ce,writeFile as Ye}from"node:fs/promises";import{dirname as qe,join as Je,relative as le,resolve as Qe}from"node:path";var N=class{constructor(e){this.root=e}root;name="local";path(e){let t=e.replace(/^\/+/,"");return Qe(this.root,t)}async put(e,t){let r=this.path(e);await He(qe(r),{recursive:!0}),await Ye(r,t)}async get(e){try{return await We(this.path(e))}catch(t){if(typeof t=="object"&&t!==null&&"code"in t&&t.code==="ENOENT")return null;throw t}}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await ae(this.path(e)),!0}catch{return!1}}async delete(e){await Ve(this.path(e),{force:!0})}async list(e=""){let t=this.path(e),r=[];try{await ae(t)}catch{return r}let o=async i=>{let c=await Ue(i,{withFileTypes:!0});for(let l of c){let a=Je(i,l.name);if(l.isDirectory()){await o(a);continue}let d=await ce(a);r.push({key:le(this.root,a),size:d.size,updatedAt:d.mtime.toISOString()})}},n=await ce(t);return n.isDirectory()?await o(t):r.push({key:le(this.root,t),size:n.size,updatedAt:n.mtime.toISOString()}),r}};import{DeleteObjectCommand as Ze,GetObjectCommand as Xe,HeadObjectCommand as et,ListObjectsV2Command as tt,PutObjectCommand as rt,S3Client as st}from"@aws-sdk/client-s3";import{getSignedUrl as ot}from"@aws-sdk/s3-request-presigner";var M=class{name;client;bucket;constructor(e){this.name=e.name??"s3",this.bucket=e.bucket,this.client=new st({region:e.region??"us-east-1",endpoint:e.endpoint||void 0,forcePathStyle:e.forcePathStyle??!1,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new rt({Bucket:this.bucket,Key:e,Body:o,ContentType:r}))}async get(e){let t=await ot(this.client,new Xe({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`${this.name} download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new et({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new Ze({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let o=await this.client.send(new tt({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of o.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=o.IsTruncated?o.NextContinuationToken:void 0}while(r);return t}};function Q(s,e){return console.warn(e),new N(s)}function de(s){let e=s.localRoot??it(nt(),".toolnet-memory","storage");if(s.provider==="r2"){let t=s.r2;return t?.accountId&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new M({name:"r2",endpoint:`https://${t.accountId}.r2.cloudflarestorage.com`,region:"auto",bucket:t.bucket,forcePathStyle:!0,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):Q(e,"[storage] Cloudflare R2 credentials missing. Using local fallback.")}if(s.provider==="s3"){let t=s.s3;return t?.bucket&&t.accessKeyId&&t.secretAccessKey?new M({name:"s3",endpoint:t.endpoint,region:t.region??"us-east-1",bucket:t.bucket,forcePathStyle:t.forcePathStyle??!1,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):Q(e,"[storage] S3 credentials missing. Using local fallback.")}if(s.provider==="huggingface"){let t=s.huggingface;return t?.namespace&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new D({namespace:t.namespace,bucket:t.bucket,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):Q(e,"[storage] Hugging Face credentials missing. Using local fallback.")}return new N(e)}var I=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"graph","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var G=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","chunks","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var K=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","vectors","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};function pe(s){let e=s.trim().replace(/\s+/g,"_").replace(/[^A-Za-z0-9._-]/g,"_").replace(/_+/g,"_").replace(/^\.+|\.+$/g,"").slice(0,100);if(!e||e==="."||e==="..")throw new Error("Invalid project storage folder");return e}function me(s){let e=s.replace(/^\/+/,"");if(e.startsWith("memories/"))return"memory/records/"+e.slice(9);if(e.startsWith("vectors/"))return"memory/vectors/"+e.slice(8);if(e.startsWith("graph/"))return"code/graph/"+e.slice(6);let t=e.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);if(t)return`${t[1]}memory/records/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/),t)return`${t[1]}memory/vectors/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/),t)return`${t[1]}code/graph/${t[2]}`;let r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);if(r)return`${r[1]}memory/records/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/),r)return`${r[1]}memory/vectors/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/),r)return`${r[1]}code/graph/${r[2]}`;let o=e.match(/^(projects\/[^/]+\/)memories\/(.+)$/);return o?`${o[1]}memory/records/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)vectors\/(.+)$/),o?`${o[1]}memory/vectors/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)graph\/(.+)$/),o?`${o[1]}code/graph/${o[2]}`:e))}var z=class{constructor(e,t,r,o){this.provider=e;this.name=e.name,this.projectId=t,this.projectName=r,this.folder=pe(o??r),this.sourcePrefix=`projects/${t}`,this.targetPrefix=`projects/${this.folder}`}provider;name;folder;sourcePrefix;targetPrefix;projectId;projectName;registryPromise;async registerProject(){let e=`${this.targetPrefix}/project.json`,t=new Date().toISOString(),r=t,o=await this.provider.getText(e);if(o){let i;try{i=JSON.parse(o)}catch(c){throw new Error(`Invalid remote ToolNet project manifest at ${e}: ${c instanceof Error?c.message:String(c)}`)}if(typeof i.id=="string"&&i.id!==this.projectId)throw new Error(["ToolNet remote project namespace collision.",`Remote folder: ${this.targetPrefix}`,`Existing project id: ${i.id}`,`Current project id: ${this.projectId}`,"Refusing to mix data from two projects."].join(" "));typeof i.createdAt=="string"&&(r=i.createdAt)}let n={version:1,id:this.projectId,name:this.projectName,remote:this.folder,createdAt:r,updatedAt:t};await this.provider.put(e,JSON.stringify(n,null,2)+`
|
|
4
|
-
`,"application/json")}async ensureRegistered(){return this.registryPromise||(this.registryPromise=this.registerProject()),this.registryPromise}key(e){if(e=
|
|
5
|
-
`).trim();if(
|
|
6
|
-
`)),ye=await o.embedMany(he);for(let b=0;b<C.length;b++){let E=C[b],w=ye[b];if(!(!E||!w||w.length===0)){if(T===void 0&&(T=w.length),w.length!==T){console.warn(`[code-semantic] skip dimension mismatch: ${w.length} != ${T}`);continue}this.vectors.upsert({id:E.id,projectId:e,vector:w,metadata:{contentHash:E.contentHash,filePath:E.filePath,symbolName:E.symbolName,symbolType:E.symbolType,startLine:E.startLine,endLine:E.endLine}}),k++}}}let Y=this.vectors.exportProject(e),te=k>0||x>0||!g||g.model!==i;return te&&await l.save({version:1,projectId:e,model:i,dimensions:Y[0]?.vector.length??T??0,updatedAt:new Date().toISOString(),records:Y}),{chunks:a.length,vectorsLoaded:S,vectorsIndexed:k,vectorsRemoved:x,totalVectors:Y.length,savedChunks:m,savedVectors:te}}async search(e,t=8){let r=await this.options.embeddings.embed(e),o=this.vectors.search(this.options.projectId,r,Math.max(t*4,20)),n=new Set(Z(e)),i=[];for(let c of o){let l=this.chunks.get(c.id);if(!l)continue;let a=new Set(Z([l.filePath,l.symbolName??"",l.symbolType??"",l.content].join(" "))),d=0;for(let x of n)a.has(x)&&d++;let m=n.size===0?0:d/n.size,g=Math.max(0,c.score),S=ue(l.filePath,e),h=(g*.75+m*.25)*S;i.push({chunk:l,score:h,vectorScore:g,lexicalScore:m})}return i.sort((c,l)=>l.score-c.score).slice(0,t)}size(){return this.chunks.size}};import*as y from"typescript";import*as p from"typescript";var _o=new Set([p.SyntaxKind.EqualsToken,p.SyntaxKind.PlusEqualsToken,p.SyntaxKind.MinusEqualsToken,p.SyntaxKind.AsteriskEqualsToken,p.SyntaxKind.SlashEqualsToken,p.SyntaxKind.PercentEqualsToken,p.SyntaxKind.QuestionQuestionEqualsToken,p.SyntaxKind.AmpersandAmpersandEqualsToken,p.SyntaxKind.BarBarEqualsToken]);var U=class{constructor(e){this.options=e}options;name="huggingface";dimensions=384;async embed(e){let[t]=await this.embedMany([e]);return t}async embedMany(e){if(e.length===0)return[];let t=`https://router.huggingface.co/hf-inference/models/${this.options.model}/pipeline/feature-extraction`,r=await fetch(t,{method:"POST",headers:{Authorization:`Bearer ${this.options.token}`,"Content-Type":"application/json"},body:JSON.stringify({inputs:e,options:{wait_for_model:!0}})});if(!r.ok)throw new Error(`HF embedding failed: ${r.status} ${await r.text()}`);let o=await r.json();if(!Array.isArray(o))throw new Error("HF embedding returned invalid response");return o.map(n=>yt(n))}};function yt(s){if(!Array.isArray(s))throw new Error("Invalid embedding");if(s.length>0&&typeof s[0]=="number")return s;if(s.length>0&&Array.isArray(s[0])){let e=s;if(e.length===0)return[];let t=e[0].length,r=new Array(t).fill(0);for(let o of e)for(let n=0;n<t;n++)r[n]+=o[n]??0;for(let o=0;o<t;o++)r[o]/=e.length;return r}throw new Error("Unsupported embedding shape")}import{createHash as St}from"node:crypto";var V=class{name="hash-fallback";dimensions=128;async embed(e){let t=new Array(this.dimensions).fill(0),r=e.toLowerCase().split(/\s+/).filter(Boolean);for(let o of r){let i=St("sha256").update(o).digest().readUInt32BE(0)%this.dimensions;t[i]+=1}return bt(t)}async embedMany(e){return Promise.all(e.map(t=>this.embed(t)))}};function bt(s){let e=Math.sqrt(s.reduce((t,r)=>t+r*r,0));return e===0?s:s.map(t=>t/e)}var X=class{constructor(e,t=new V){this.primary=e;this.fallback=t}primary;fallback;name="resilient";async embed(e){if(this.primary)try{return await this.primary.embed(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embed(e)}async embedMany(e){if(this.primary)try{return await this.primary.embedMany(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embedMany(e)}};function ge(){let s=process.env.HF_TOKEN,e=process.env.HF_EMBEDDING_MODEL??"sentence-transformers/all-MiniLM-L6-v2",t=s?new U({token:s,model:e}):null;return new X(t)}async function xt(){let s=re(),e=new _().detect(),t=de({provider:s.storage.provider,huggingface:s.storage.huggingface,localRoot:s.storage.localRoot}),r=new z(t,e.id,e.name,e.remote??e.name),o=new v,n=await new I(r).load(e.id);if(!n)throw new Error("Code graph missing. Run npm run code:index:test first.");o.import(n.symbols,n.edges);let i=new W({projectId:e.id,rootPath:e.rootPath,model:process.env.HF_EMBEDDING_MODEL??"sentence-transformers/all-MiniLM-L6-v2",storage:r,embeddings:ge(),graph:o}),c=await i.initialize(),l=process.argv.slice(2).join(" ")||"memory storage hugging face",a=await i.search(l,5);console.log({ok:!0,query:l,...c,results:a.map(d=>({score:Number(d.score.toFixed(4)),file:d.chunk.filePath,symbol:d.chunk.symbolName,lines:`${d.chunk.startLine}-${d.chunk.endLine}`,preview:d.chunk.content.replace(/\s+/g," ").slice(0,140)}))})}xt().catch(s=>{console.error(s),process.exit(1)});
|
|
1
|
+
import"dotenv/config";import{existsSync as Re,readFileSync as ke}from"node:fs";import{homedir as Ie}from"node:os";import{join as Te}from"node:path";function _e(r){let e=r.trim();return e.length>=2&&e.startsWith('"')&&e.endsWith('"')?(e=e.slice(1,-1),e.replace(/\\n/g,`
|
|
2
|
+
`).replace(/\\r/g,"\r").replace(/\\t/g," ").replace(/\\"/g,'"').replace(/\\\\/g,"\\")):e.length>=2&&e.startsWith("'")&&e.endsWith("'")?e.slice(1,-1):e}function Me(){let r=process.env.TOOLNET_GLOBAL_ENV??Te(Ie(),".config","toolnet-memory",".env");if(!Re(r))return;let e=ke(r,"utf8");for(let t of e.split(/\r?\n/)){let s=t.trim();if(!s||s.startsWith("#"))continue;s.startsWith("export ")&&(s=s.slice(7));let o=s.indexOf("=");if(o<=0)continue;let n=s.slice(0,o).trim();/^[A-Za-z_][A-Za-z0-9_]*$/.test(n)&&process.env[n]===void 0&&(process.env[n]=_e(s.slice(o+1)))}}Me();function I(r,e){return r===void 0?e:["1","true","yes","on"].includes(r.toLowerCase())}function T(r,e){if(!r)return e;let t=Number(r);return Number.isFinite(t)?t:e}function le(){return{memory:{autoCapture:I(process.env.MEMORY_AUTO_CAPTURE,!0),autoRetrieve:I(process.env.MEMORY_AUTO_RETRIEVE,!0),autoSummarize:I(process.env.MEMORY_AUTO_SUMMARIZE,!0),autoSync:I(process.env.MEMORY_AUTO_SYNC,!0)},retrieval:{maxCandidates:T(process.env.MEMORY_MAX_CANDIDATES,50),rerankTop:T(process.env.MEMORY_RERANK_TOP,10),finalContext:T(process.env.MEMORY_FINAL_CONTEXT,5),tokenBudget:T(process.env.MEMORY_TOKEN_BUDGET,2e3)},storage:{provider:process.env.MEMORY_STORAGE_PROVIDER??"huggingface",r2:{accountId:process.env.R2_ACCOUNT_ID,bucket:process.env.R2_BUCKET,accessKeyId:process.env.R2_ACCESS_KEY_ID,secretAccessKey:process.env.R2_SECRET_ACCESS_KEY},s3:{endpoint:process.env.S3_ENDPOINT,region:process.env.S3_REGION,bucket:process.env.S3_BUCKET,accessKeyId:process.env.S3_ACCESS_KEY_ID,secretAccessKey:process.env.S3_SECRET_ACCESS_KEY,forcePathStyle:I(process.env.S3_FORCE_PATH_STYLE,!1)},huggingface:{namespace:process.env.HF_NAMESPACE,bucket:process.env.HF_BUCKET,accessKeyId:process.env.HF_S3_ACCESS_KEY_ID,secretAccessKey:process.env.HF_S3_SECRET_ACCESS_KEY},localRoot:process.env.MEMORY_LOCAL_STORAGE_PATH},cache:{maxMb:T(process.env.MEMORY_LOCAL_CACHE_MB,200)}}}import{createHash as Ne}from"node:crypto";import{existsSync as te,mkdirSync as Oe,readFileSync as je,renameSync as De,writeFileSync as Fe}from"node:fs";import{basename as Ke,dirname as D,join as K,parse as ue,resolve as _}from"node:path";var me=".toolnet",$e="project.json";function Ge(r){return Ne("sha256").update(r).digest("hex").slice(0,16)}function re(r){return K(r,me,$e)}function Ue(r){return te(re(r))}function Be(r,e){let t=_(r),s=ue(t).root;for(;;){if(Ue(t))return t;if(t===s||e&&t===_(e))break;let o=D(t);if(o===t)break;t=o}return null}function ze(r){let e=_(r),t=ue(e).root,s=[".git","package.json","pyproject.toml","Cargo.toml","go.mod","composer.json"];for(;;){if(s.some(n=>te(K(e,n))))return e;if(e===t)break;let o=D(e);if(o===e)break;e=o}return _(r)}function He(r){let e;try{e=JSON.parse(je(r,"utf8"))}catch(o){throw new Error(`Invalid ToolNet project manifest: ${r}: ${o instanceof Error?o.message:String(o)}`)}if(!e||typeof e!="object")throw new Error(`Invalid ToolNet project manifest: ${r}`);let t=e;if(typeof t.id!="string"||!t.id.trim())throw new Error(`ToolNet project manifest is missing id: ${r}`);if(typeof t.name!="string"||!t.name.trim())throw new Error(`ToolNet project manifest is missing name: ${r}`);let s=new Date().toISOString();return{version:1,id:t.id,name:t.name,remote:typeof t.remote=="string"&&t.remote.trim()?t.remote:t.name,rootPath:typeof t.rootPath=="string"?t.rootPath:D(D(r)),createdAt:typeof t.createdAt=="string"?t.createdAt:s,updatedAt:typeof t.updatedAt=="string"?t.updatedAt:s,graphVersion:typeof t.graphVersion=="number"?t.graphVersion:0,memoryVersion:typeof t.memoryVersion=="number"?t.memoryVersion:0,metadata:t.metadata&&typeof t.metadata=="object"?t.metadata:void 0}}function de(r,e){let t=K(r,me);Oe(t,{recursive:!0});let s=re(r),o=`${s}.tmp-${process.pid}`;Fe(o,JSON.stringify(e,null,2)+`
|
|
3
|
+
`,{encoding:"utf8",mode:384}),De(o,s)}function pe(r,e){return{id:r.id,name:r.name,remote:r.remote,rootPath:e,createdAt:r.createdAt,updatedAt:r.updatedAt,graphVersion:r.graphVersion,memoryVersion:r.memoryVersion,metadata:r.metadata}}var F=class{detect(e=process.cwd()){let t=_(e),s=ze(t),n=[".git","package.json","pyproject.toml","Cargo.toml","go.mod","composer.json"].some(p=>te(K(s,p))),a=Be(t,n?s:void 0);if(a){let p=re(a),f=He(p);return f.rootPath!==a&&(f.rootPath=a,f.updatedAt=new Date().toISOString(),de(a,f)),pe(f,a)}let c=new Date().toISOString(),l=Ke(s),d={version:1,id:Ge(s),name:l,remote:l,rootPath:s,createdAt:c,updatedAt:c,graphVersion:0,memoryVersion:0};return de(s,d),pe(d,s)}};var Ve=[{type:"openai_key",regex:/\bsk-[A-Za-z0-9_\-]{20,}\b/g},{type:"huggingface_token",regex:/\bhf_[A-Za-z0-9]{20,}\b/g},{type:"hf_s3_access_key",regex:/\bHFAK[A-Za-z0-9]{8,}\b/g},{type:"bearer_token",regex:/\bBearer\s+[A-Za-z0-9._\-]{16,}\b/gi},{type:"jwt",regex:/\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\b/g},{type:"private_key",regex:/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g},{type:"password_assignment",regex:/\b(password|passwd|pwd)\s*[:=]\s*["']?[^"'\s]{6,}["']?/gi},{type:"secret_assignment",regex:/\b(secret|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*["']?[^"'\s]{8,}["']?/gi},{type:"cookie",regex:/\b(cookie|set-cookie)\s*[:=]\s*[^;\n]{8,}/gi}],$=class{scan(e){let t=[];for(let s of Ve){let o=new RegExp(s.regex.source,s.regex.flags);for(let n of e.matchAll(o))t.push({type:s.type,value:n[0]})}return t}hasSecrets(e){return this.scan(e).length>0}};var M=class{scanner=new $;sanitize(e){let t=e,s=this.scanner.scan(e),o=new Set;for(let n of s)o.add(n.type),t=t.split(n.value).join(`[REDACTED:${n.type}]`);return{text:t,redacted:s.length,secretTypes:[...o]}}sanitizeValue(e){if(typeof e=="string")return this.sanitize(e).text;if(Array.isArray(e))return e.map(t=>this.sanitizeValue(t));if(e&&typeof e=="object"){let t={};for(let[s,o]of Object.entries(e)){let n=s.normalize("NFKC").toLowerCase().replace(/[^a-z0-9]+/g,"");n.includes("password")||n.includes("passwd")||n==="pwd"||n.includes("secret")||n.includes("token")||n.includes("cookie")||n.includes("authorization")||n.includes("apikey")||n.includes("accesskey")||n.includes("privatekey")||n.includes("clientsecret")||n.includes("credential")?t[s]="[REDACTED]":t[s]=this.sanitizeValue(o)}return t}return e}};import{homedir as gt}from"node:os";import{join as ht}from"node:path";import{DeleteObjectCommand as We,GetObjectCommand as Ye,HeadObjectCommand as qe,ListObjectsV2Command as Je,PutObjectCommand as Qe,S3Client as Ze}from"@aws-sdk/client-s3";import{getSignedUrl as Xe}from"@aws-sdk/s3-request-presigner";var G=class{name="huggingface";client;bucket;constructor(e){this.bucket=e.bucket,this.client=new Ze({region:"us-east-1",endpoint:`https://s3.hf.co/${e.namespace}`,forcePathStyle:!0,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,s="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new Qe({Bucket:this.bucket,Key:e,Body:o,ContentType:s}))}async get(e){let t=await Xe(this.client,new Ye({Bucket:this.bucket,Key:e}),{expiresIn:60}),s=await fetch(t,{redirect:"follow"});if(s.status===404)return null;if(!s.ok)throw new Error(`HF download failed: ${s.status} ${s.statusText}`);return new Uint8Array(await s.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new qe({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new We({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],s;do{let o=await this.client.send(new Je({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:s}));for(let n of o.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});s=o.IsTruncated?o.NextContinuationToken:void 0}while(s);return t}};import{access as fe,mkdir as et,readFile as tt,readdir as rt,rm as st,stat as ge,writeFile as ot}from"node:fs/promises";import{dirname as nt,join as it,relative as he,resolve as at}from"node:path";var N=class{constructor(e){this.root=e}root;name="local";path(e){let t=e.replace(/^\/+/,"");return at(this.root,t)}async put(e,t){let s=this.path(e);await et(nt(s),{recursive:!0}),await ot(s,t)}async get(e){try{return await tt(this.path(e))}catch(t){if(typeof t=="object"&&t!==null&&"code"in t&&t.code==="ENOENT")return null;throw t}}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await fe(this.path(e)),!0}catch{return!1}}async delete(e){await st(this.path(e),{force:!0})}async list(e=""){let t=this.path(e),s=[];try{await fe(t)}catch{return s}let o=async a=>{let c=await rt(a,{withFileTypes:!0});for(let l of c){let d=it(a,l.name);if(l.isDirectory()){await o(d);continue}let p=await ge(d);s.push({key:he(this.root,d),size:p.size,updatedAt:p.mtime.toISOString()})}},n=await ge(t);return n.isDirectory()?await o(t):s.push({key:he(this.root,t),size:n.size,updatedAt:n.mtime.toISOString()}),s}};import{DeleteObjectCommand as ct,GetObjectCommand as lt,HeadObjectCommand as dt,ListObjectsV2Command as pt,PutObjectCommand as ut,S3Client as mt}from"@aws-sdk/client-s3";import{getSignedUrl as ft}from"@aws-sdk/s3-request-presigner";var O=class{name;client;bucket;constructor(e){this.name=e.name??"s3",this.bucket=e.bucket,this.client=new mt({region:e.region??"us-east-1",endpoint:e.endpoint||void 0,forcePathStyle:e.forcePathStyle??!1,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,s="application/octet-stream"){let o=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new ut({Bucket:this.bucket,Key:e,Body:o,ContentType:s}))}async get(e){let t=await ft(this.client,new lt({Bucket:this.bucket,Key:e}),{expiresIn:60}),s=await fetch(t,{redirect:"follow"});if(s.status===404)return null;if(!s.ok)throw new Error(`${this.name} download failed: ${s.status} ${s.statusText}`);return new Uint8Array(await s.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new dt({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new ct({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],s;do{let o=await this.client.send(new pt({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:s}));for(let n of o.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});s=o.IsTruncated?o.NextContinuationToken:void 0}while(s);return t}};function se(r,e){return console.warn(e),new N(r)}function ye(r){let e=r.localRoot??ht(gt(),".toolnet-memory","storage");if(r.provider==="r2"){let t=r.r2;return t?.accountId&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new O({name:"r2",endpoint:`https://${t.accountId}.r2.cloudflarestorage.com`,region:"auto",bucket:t.bucket,forcePathStyle:!0,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):se(e,"[storage] Cloudflare R2 credentials missing. Using local fallback.")}if(r.provider==="s3"){let t=r.s3;return t?.bucket&&t.accessKeyId&&t.secretAccessKey?new O({name:"s3",endpoint:t.endpoint,region:t.region??"us-east-1",bucket:t.bucket,forcePathStyle:t.forcePathStyle??!1,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):se(e,"[storage] S3 credentials missing. Using local fallback.")}if(r.provider==="huggingface"){let t=r.huggingface;return t?.namespace&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new G({namespace:t.namespace,bucket:t.bucket,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):se(e,"[storage] Hugging Face credentials missing. Using local fallback.")}return new N(e)}var j=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"graph","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var U=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","chunks","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};var B=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"code","vectors","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};function be(r){let e=r.trim().replace(/\s+/g,"_").replace(/[^A-Za-z0-9._-]/g,"_").replace(/_+/g,"_").replace(/^\.+|\.+$/g,"").slice(0,100);if(!e||e==="."||e==="..")throw new Error("Invalid project storage folder");return e}function Se(r){let e=r.replace(/^\/+/,"");if(e.startsWith("memories/"))return"memory/records/"+e.slice(9);if(e.startsWith("vectors/"))return"memory/vectors/"+e.slice(8);if(e.startsWith("graph/"))return"code/graph/"+e.slice(6);let t=e.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);if(t)return`${t[1]}memory/records/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/),t)return`${t[1]}memory/vectors/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/),t)return`${t[1]}code/graph/${t[2]}`;let s=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);if(s)return`${s[1]}memory/records/${s[2]}`;if(s=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/),s)return`${s[1]}memory/vectors/${s[2]}`;if(s=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/),s)return`${s[1]}code/graph/${s[2]}`;let o=e.match(/^(projects\/[^/]+\/)memories\/(.+)$/);return o?`${o[1]}memory/records/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)vectors\/(.+)$/),o?`${o[1]}memory/vectors/${o[2]}`:(o=e.match(/^(projects\/[^/]+\/)graph\/(.+)$/),o?`${o[1]}code/graph/${o[2]}`:e))}var z=class{constructor(e,t,s,o){this.provider=e;this.name=e.name,this.projectId=t,this.projectName=s,this.folder=be(o??s),this.sourcePrefix=`projects/${t}`,this.targetPrefix=`projects/${this.folder}`}provider;name;folder;sourcePrefix;targetPrefix;projectId;projectName;registryPromise;async registerProject(){let e=`${this.targetPrefix}/project.json`,t=new Date().toISOString(),s=t,o=await this.provider.getText(e);if(o){let a;try{a=JSON.parse(o)}catch(c){throw new Error(`Invalid remote ToolNet project manifest at ${e}: ${c instanceof Error?c.message:String(c)}`)}if(typeof a.id=="string"&&a.id!==this.projectId)throw new Error(["ToolNet remote project namespace collision.",`Remote folder: ${this.targetPrefix}`,`Existing project id: ${a.id}`,`Current project id: ${this.projectId}`,"Refusing to mix data from two projects."].join(" "));typeof a.createdAt=="string"&&(s=a.createdAt)}let n={version:1,id:this.projectId,name:this.projectName,remote:this.folder,createdAt:s,updatedAt:t};await this.provider.put(e,JSON.stringify(n,null,2)+`
|
|
4
|
+
`,"application/json")}async ensureRegistered(){return this.registryPromise||(this.registryPromise=this.registerProject()),this.registryPromise}key(e){if(e=Se(e),e===this.sourcePrefix)return this.targetPrefix;if(e.startsWith(`${this.sourcePrefix}/`))return this.targetPrefix+e.slice(this.sourcePrefix.length);if(e===this.targetPrefix||e.startsWith(`${this.targetPrefix}/`))return e;if(e.startsWith("projects/"))throw new Error(["Cross-project storage access denied.",`Current project: ${this.targetPrefix}`,`Requested key: ${e}`].join(" "));return e}async put(e,t,s){return await this.ensureRegistered(),this.provider.put(this.key(e),t,s)}async get(e){return await this.ensureRegistered(),this.provider.get(this.key(e))}async getText(e){return await this.ensureRegistered(),this.provider.getText(this.key(e))}async delete(e){return await this.ensureRegistered(),this.provider.delete(this.key(e))}async exists(e){return await this.ensureRegistered(),this.provider.exists(this.key(e))}async list(e){return await this.ensureRegistered(),this.provider.list(this.key(e))}};import*as h from"typescript";var A=class{symbols=new Map;edges=new Map;addSymbol(e){this.symbols.set(e.id,e)}addEdge(e){this.edges.set(e.id,e)}getSymbol(e){return this.symbols.get(e)}findByName(e,t){return[...this.symbols.values()].filter(s=>s.projectId===e&&(s.name===t||s.qualifiedName===t))}allSymbols(e){return[...this.symbols.values()].filter(t=>t.projectId===e)}allEdges(e){return[...this.edges.values()].filter(t=>t.projectId===e)}import(e,t){for(let s of e)this.addSymbol(s);for(let s of t)this.addEdge(s)}clearProject(e){for(let[t,s]of this.symbols)s.projectId===e&&this.symbols.delete(t);for(let[t,s]of this.edges)s.projectId===e&&this.edges.delete(t)}};import{execFile as Et}from"node:child_process";import{promisify as Ct}from"node:util";var Qs=Ct(Et);import{createHash as vt}from"node:crypto";import{readFile as xt}from"node:fs/promises";import{join as wt}from"node:path";function Ee(r){return vt("sha256").update(r).digest("hex")}var H=class{sanitizer=new M;async build(e,t,s,o={}){let n=o.maxLines??80,a=o.overlapLines??10,c=new Map;for(let d of s){let p=c.get(d.filePath)??[];p.push(d),c.set(d.filePath,p)}let l=[];for(let[d,p]of c){let f;try{f=await xt(wt(t,d),"utf8")}catch{continue}let y=f.split(/\r?\n/),E=p.filter(b=>b.type!=="file"&&typeof b.startLine=="number"&&typeof b.endLine=="number");if(E.length>0)for(let b of E)l.push(...this.chunkRange({projectId:e,filePath:d,lines:y,symbol:b,startLine:b.startLine,endLine:b.endLine,maxLines:n,overlapLines:a}));else l.push(...this.chunkRange({projectId:e,filePath:d,lines:y,startLine:1,endLine:Math.max(1,y.length),maxLines:n,overlapLines:a}))}return l}chunkRange(e){let{projectId:t,filePath:s,lines:o,symbol:n,maxLines:a}=e,c=Math.min(e.overlapLines,Math.max(0,a-1)),l=Math.max(1,e.startLine),d=Math.min(o.length,Math.max(l,e.endLine)),p=[],f=l,y=0;for(;f<=d;){let E=Math.min(d,f+a-1),b=o.slice(f-1,E).join(`
|
|
5
|
+
`).trim();if(b){let v=this.sanitizer.sanitize(b).text,L=Ee(v),R=[t,s,n?.id??"file",String(y),L].join(":");p.push({id:Ee(R).slice(0,32),projectId:t,filePath:s,symbolId:n?.id,symbolName:n?.qualifiedName??n?.name,symbolType:n?.type,startLine:f,endLine:E,content:v,contentHash:L})}if(E>=d)break;f=E-c+1,y++}return p}};var V=class{records=new Map;upsert(e){this.records.set(e.id,e)}get(e){return this.records.get(e)}has(e){return this.records.has(e)}remove(e){return this.records.delete(e)}clearProject(e){let t=0;for(let[s,o]of this.records)o.projectId===e&&(this.records.delete(s),t++);return t}exportProject(e){return[...this.records.values()].filter(t=>t.projectId===e)}importRecords(e){let t=0;for(let s of e)!s?.id||!s?.projectId||!Array.isArray(s.vector)||(this.records.set(s.id,s),t++);return t}search(e,t,s=10){let o=[];for(let n of this.records.values()){if(n.projectId!==e)continue;let a=Pt(t,n.vector);o.push({id:n.id,score:a,metadata:n.metadata})}return o.sort((n,a)=>a.score-n.score).slice(0,s)}size(){return this.records.size}};function Pt(r,e){if(r.length===0||e.length===0||r.length!==e.length)return 0;let t=0,s=0,o=0;for(let n=0;n<r.length;n++){let a=r[n]??0,c=e[n]??0;t+=a*c,s+=a*a,o+=c*c}return s===0||o===0?0:t/(Math.sqrt(s)*Math.sqrt(o))}function oe(r){return r.toLowerCase().normalize("NFKC").replace(/[^\p{L}\p{N}_\-./]+/gu," ").split(/\s+/).map(e=>e.trim()).filter(e=>e.length>1)}function Ce(r,e){let t=r.toLowerCase().replaceAll("\\","/"),s=e.toLowerCase(),o=/\b(test|tests|spec|fixture|mock)\b/.test(s);return(t.startsWith("tests/")||t.includes("/tests/")||t.includes("__tests__")||/\.(test|spec)\.[^.]+$/.test(t))&&!o?.35:t.includes("/generated/")||t.includes(".generated.")?.25:1}var W=class{constructor(e){this.options=e}options;vectors=new V;chunks=new Map;async initialize(){let{projectId:e,rootPath:t,storage:s,embeddings:o,graph:n,model:a}=this.options,c=new U(s),l=new B(s),d=await new H().build(e,t,n.allSymbols(e));this.chunks.clear();for(let g of d)this.chunks.set(g.id,g);let p=await c.load(e),f=!p||p.chunks.length!==d.length||p.chunks.some(g=>!this.chunks.has(g.id));f&&await c.save({version:1,projectId:e,updatedAt:new Date().toISOString(),chunks:d});let y=await l.load(e),E=0;y&&y.model===a&&(E=this.vectors.importRecords(y.records));let b=new Set(d.map(g=>g.id)),v=0;for(let g of this.vectors.exportProject(e))b.has(g.id)||this.vectors.remove(g.id)&&v++;let L=d.filter(g=>{let w=this.vectors.get(g.id);return w?w.metadata?.contentHash!==g.contentHash:!0}),R=0,ae=8,k=this.vectors.exportProject(e)[0]?.vector.length;for(let g=0;g<L.length;g+=ae){let w=L.slice(g,g+ae),Ae=w.map(C=>[`file: ${C.filePath}`,C.symbolName?`symbol: ${C.symbolName}`:"",C.symbolType?`type: ${C.symbolType}`:"",C.content].filter(Boolean).join(`
|
|
6
|
+
`)),Le=await o.embedMany(Ae);for(let C=0;C<w.length;C++){let x=w[C],P=Le[C];if(!(!x||!P||P.length===0)){if(k===void 0&&(k=P.length),P.length!==k){console.warn(`[code-semantic] skip dimension mismatch: ${P.length} != ${k}`);continue}this.vectors.upsert({id:x.id,projectId:e,vector:P,metadata:{contentHash:x.contentHash,filePath:x.filePath,symbolName:x.symbolName,symbolType:x.symbolType,startLine:x.startLine,endLine:x.endLine}}),R++}}}let ee=this.vectors.exportProject(e),ce=R>0||v>0||!y||y.model!==a;return ce&&await l.save({version:1,projectId:e,model:a,dimensions:ee[0]?.vector.length??k??0,updatedAt:new Date().toISOString(),records:ee}),{chunks:d.length,vectorsLoaded:E,vectorsIndexed:R,vectorsRemoved:v,totalVectors:ee.length,savedChunks:f,savedVectors:ce}}async search(e,t=8){let s=await this.options.embeddings.embed(e),o=this.vectors.search(this.options.projectId,s,Math.max(t*4,20)),n=new Set(oe(e)),a=[];for(let c of o){let l=this.chunks.get(c.id);if(!l)continue;let d=new Set(oe([l.filePath,l.symbolName??"",l.symbolType??"",l.content].join(" "))),p=0;for(let v of n)d.has(v)&&p++;let f=n.size===0?0:p/n.size,y=Math.max(0,c.score),E=Ce(l.filePath,e),b=(y*.75+f*.25)*E;a.push({chunk:l,score:b,vectorScore:y,lexicalScore:f})}return a.sort((c,l)=>l.score-c.score).slice(0,t)}size(){return this.chunks.size}};import*as S from"typescript";import*as u from"typescript";var on=new Set([u.SyntaxKind.EqualsToken,u.SyntaxKind.PlusEqualsToken,u.SyntaxKind.MinusEqualsToken,u.SyntaxKind.AsteriskEqualsToken,u.SyntaxKind.SlashEqualsToken,u.SyntaxKind.PercentEqualsToken,u.SyntaxKind.QuestionQuestionEqualsToken,u.SyntaxKind.AmpersandAmpersandEqualsToken,u.SyntaxKind.BarBarEqualsToken]);var ve=[{id:"openai-compatible",label:"OpenAI-compatible",requiresApiKey:!0,requiresBaseUrl:!0,transport:"openai-compatible"},{id:"alibaba",label:"Alibaba / DashScope",requiresApiKey:!0,requiresBaseUrl:!0,transport:"openai-compatible"},{id:"openrouter",label:"OpenRouter",defaultBaseUrl:"https://openrouter.ai/api/v1",requiresApiKey:!0,transport:"openai-compatible"},{id:"groq",label:"Groq",defaultBaseUrl:"https://api.groq.com/openai/v1",requiresApiKey:!0,transport:"openai-compatible"},{id:"deepseek",label:"DeepSeek",defaultBaseUrl:"https://api.deepseek.com",defaultModel:"deepseek-v4-flash",requiresApiKey:!0,transport:"openai-compatible"},{id:"nvidia",label:"NVIDIA NIM",defaultBaseUrl:"https://integrate.api.nvidia.com/v1",defaultModel:"deepseek-ai/deepseek-v4-pro",requiresApiKey:!0,transport:"openai-compatible"},{id:"gemini",label:"Gemini",defaultBaseUrl:"https://generativelanguage.googleapis.com/v1beta",requiresApiKey:!0,transport:"gemini"},{id:"huggingface",label:"Hugging Face",defaultBaseUrl:"https://router.huggingface.co/v1",requiresApiKey:!0,transport:"openai-compatible"},{id:"ollama",label:"Ollama / Local",defaultBaseUrl:"http://127.0.0.1:11434/v1",requiresApiKey:!1,transport:"openai-compatible"},{id:"custom",label:"Custom endpoint",requiresApiKey:!1,requiresBaseUrl:!0,transport:"openai-compatible"},{id:"cloudflare",label:"Cloudflare Workers AI",requiresApiKey:!0,requiresAccountId:!0,transport:"cloudflare"}];function Y(r){let e=ve.find(t=>t.id===r);if(!e)throw new Error(`Unsupported AI provider: ${r}`);return e}function q(r){return ve.some(e=>e.id===r)}function i(r){return process.env[r]?.trim()||void 0}function m(...r){return r.find(e=>!!e?.trim())}function xe(){if(i("GROQ_API_KEY"))return"groq";if(i("DEEPSEEK_API_KEY"))return"deepseek";if(i("NVIDIA_API_KEY")||i("NVIDIA_NIM_API_KEY"))return"nvidia";if(i("OPENROUTER_API_KEY"))return"openrouter";if(i("ALIBABA_API_KEY")||i("DASHSCOPE_API_KEY"))return"alibaba";if(i("GEMINI_API_KEY")||i("GOOGLE_API_KEY"))return"gemini";if(i("CLOUDFLARE_API_TOKEN")&&i("CLOUDFLARE_ACCOUNT_ID"))return"cloudflare";if(i("HF_TOKEN"))return"huggingface";if(i("OLLAMA_MODEL")||i("OLLAMA_BASE_URL"))return"ollama"}function Lt(){let r=i("TOOLNET_LLM_PROVIDER");return r&&q(r)?r:xe()??"openai-compatible"}function ne(r){let e=Y(r);switch(r){case"alibaba":return{provider:r,apiKey:m(i("ALIBABA_API_KEY"),i("DASHSCOPE_API_KEY")),baseUrl:m(i("ALIBABA_BASE_URL"),i("DASHSCOPE_BASE_URL"),e.defaultBaseUrl),model:m(i("ALIBABA_MODEL"),i("DASHSCOPE_MODEL"))};case"openrouter":return{provider:r,apiKey:i("OPENROUTER_API_KEY"),baseUrl:m(i("OPENROUTER_BASE_URL"),e.defaultBaseUrl),model:i("OPENROUTER_MODEL")};case"groq":return{provider:r,apiKey:i("GROQ_API_KEY"),baseUrl:m(i("GROQ_BASE_URL"),e.defaultBaseUrl),model:i("GROQ_MODEL")};case"deepseek":return{provider:r,apiKey:i("DEEPSEEK_API_KEY"),baseUrl:m(i("DEEPSEEK_BASE_URL"),e.defaultBaseUrl),model:m(i("DEEPSEEK_MODEL"),e.defaultModel)};case"nvidia":return{provider:r,apiKey:m(i("NVIDIA_API_KEY"),i("NVIDIA_NIM_API_KEY")),baseUrl:m(i("NVIDIA_BASE_URL"),i("NVIDIA_NIM_BASE_URL"),e.defaultBaseUrl),model:m(i("NVIDIA_MODEL"),i("NVIDIA_NIM_MODEL"),e.defaultModel)};case"gemini":return{provider:r,apiKey:m(i("GEMINI_API_KEY"),i("GOOGLE_API_KEY")),baseUrl:m(i("GEMINI_BASE_URL"),e.defaultBaseUrl),model:i("GEMINI_MODEL")};case"huggingface":return{provider:r,apiKey:i("HF_TOKEN"),baseUrl:m(i("HF_INFERENCE_BASE_URL"),e.defaultBaseUrl),model:m(i("HF_LLM_MODEL"),i("HF_MODEL"))};case"ollama":return{provider:r,apiKey:i("OLLAMA_API_KEY"),baseUrl:m(i("OLLAMA_BASE_URL"),e.defaultBaseUrl),model:i("OLLAMA_MODEL")};case"cloudflare":return{provider:r,accountId:i("CLOUDFLARE_ACCOUNT_ID"),apiKey:i("CLOUDFLARE_API_TOKEN"),baseUrl:i("CLOUDFLARE_AI_BASE_URL"),model:i("CLOUDFLARE_MODEL")};case"custom":return{provider:r,apiKey:i("CUSTOM_AI_API_KEY"),baseUrl:i("CUSTOM_AI_BASE_URL"),model:i("CUSTOM_AI_MODEL")};default:return{provider:"openai-compatible",apiKey:m(i("OPENAI_API_KEY"),i("MODEL_API_KEY")),baseUrl:m(i("OPENAI_BASE_URL"),i("MODEL_BASE_URL")),model:m(i("OPENAI_MODEL"),i("MODEL_NAME"))}}}function Rt(){let r=i("TOOLNET_LLM_PROVIDER"),e=r&&q(r)?r:Lt(),t=Y(e),s=ne(e);return{provider:e,apiKey:m(i("TOOLNET_LLM_API_KEY"),s.apiKey),baseUrl:m(i("TOOLNET_LLM_BASE_URL"),s.baseUrl,t.defaultBaseUrl),model:m(i("TOOLNET_LLM_MODEL"),s.model,t.defaultModel),accountId:m(i("TOOLNET_LLM_ACCOUNT_ID"),s.accountId)}}function kt(){let r=i("TOOLNET_EMBEDDING_PROVIDER");return r==="local"?"local":r&&q(r)?r:i("HF_TOKEN")||i("HF_EMBEDDING_MODEL")?"huggingface":"local"}function It(){let r=kt();if(r==="local")return{provider:"local",model:m(i("TOOLNET_EMBEDDING_MODEL"),i("LOCAL_EMBEDDING_MODEL"))};let e=Y(r),t,s,o,n;switch(r){case"huggingface":t=i("HF_TOKEN"),s=i("HF_INFERENCE_BASE_URL"),o=i("HF_EMBEDDING_MODEL");break;case"openai-compatible":t=i("OPENAI_API_KEY"),s=i("OPENAI_BASE_URL"),o=i("OPENAI_EMBEDDING_MODEL");break;case"cloudflare":t=i("CLOUDFLARE_API_TOKEN"),s=i("CLOUDFLARE_AI_BASE_URL"),o=i("CLOUDFLARE_EMBEDDING_MODEL"),n=i("CLOUDFLARE_ACCOUNT_ID");break;default:t=ne(r).apiKey,s=ne(r).baseUrl,o=i(`${r.toUpperCase().replace(/-/g,"_")}_EMBEDDING_MODEL`)}let a=m(i("TOOLNET_EMBEDDING_ACCOUNT_ID"),n),c=m(i("TOOLNET_EMBEDDING_BASE_URL"),s,e.defaultBaseUrl),l=r==="cloudflare"&&!c&&a?`https://api.cloudflare.com/client/v4/accounts/${a}/ai/v1`:c;return{provider:r,apiKey:m(i("TOOLNET_EMBEDDING_API_KEY"),t),baseUrl:l,model:m(i("TOOLNET_EMBEDDING_MODEL"),o),accountId:a}}function we(){let r=Rt(),e=It();return{llm:r,embedding:e,legacy:{llm:!i("TOOLNET_LLM_PROVIDER")&&!!xe(),embedding:!i("TOOLNET_EMBEDDING_PROVIDER")&&!!(i("HF_TOKEN")||i("HF_EMBEDDING_MODEL"))}}}var J=class{constructor(e){this.options=e}options;name="gemini";async embed(e){let t=this.options.model.replace(/^models\//,""),o=`${this.options.baseUrl?.replace(/\/+$/,"")||"https://generativelanguage.googleapis.com/v1beta"}/models/${encodeURIComponent(t)}:embedContent?key=${encodeURIComponent(this.options.apiKey)}`,n=await fetch(o,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({model:`models/${t}`,content:{parts:[{text:e}]}})});if(!n.ok)throw new Error(`Gemini embedding failed: ${n.status} ${await n.text()}`);let c=(await n.json()).embedding?.values;if(!Array.isArray(c)||c.length===0)throw new Error("Gemini embedding returned invalid response");return c}async embedMany(e){let t=[];for(let s of e)t.push(await this.embed(s));return t}};import{createHash as Tt}from"node:crypto";var Q=class{name="hash-fallback";dimensions=128;async embed(e){let t=new Array(this.dimensions).fill(0),s=e.toLowerCase().split(/\s+/).filter(Boolean);for(let o of s){let a=Tt("sha256").update(o).digest().readUInt32BE(0)%this.dimensions;t[a]+=1}return _t(t)}async embedMany(e){return Promise.all(e.map(t=>this.embed(t)))}};function _t(r){let e=Math.sqrt(r.reduce((t,s)=>t+s*s,0));return e===0?r:r.map(t=>t/e)}var Z=class{constructor(e){this.options=e;this.dimensions=e.dimensions}options;name="openai-compatible";dimensions;async embed(e){let[t]=await this.embedMany([e]);return t}async embedMany(e){if(e.length===0)return[];let t=this.options.baseUrl.replace(/\/+$/,""),s=await fetch(`${t}/embeddings`,{method:"POST",headers:{"content-type":"application/json",...this.options.apiKey?{authorization:`Bearer ${this.options.apiKey}`}:{}},body:JSON.stringify({model:this.options.model,input:e})});if(!s.ok)throw new Error(`Embedding failed: ${s.status} ${await s.text()}`);let n=(await s.json()).data?.sort((a,c)=>(a.index??0)-(c.index??0)).map(a=>a.embedding);if(!n||n.length!==e.length||n.some(a=>!Array.isArray(a)))throw new Error("Embedding provider returned invalid response");return n}};var X=class{constructor(e){this.options=e}options;name="huggingface";dimensions=384;async embed(e){let[t]=await this.embedMany([e]);return t}async embedMany(e){if(e.length===0)return[];let t=`https://router.huggingface.co/hf-inference/models/${this.options.model}/pipeline/feature-extraction`,s=await fetch(t,{method:"POST",headers:{Authorization:`Bearer ${this.options.token}`,"Content-Type":"application/json"},body:JSON.stringify({inputs:e,options:{wait_for_model:!0}})});if(!s.ok)throw new Error(`HF embedding failed: ${s.status} ${await s.text()}`);let o=await s.json();if(!Array.isArray(o))throw new Error("HF embedding returned invalid response");return o.map(n=>Mt(n))}};function Mt(r){if(!Array.isArray(r))throw new Error("Invalid embedding");if(r.length>0&&typeof r[0]=="number")return r;if(r.length>0&&Array.isArray(r[0])){let e=r;if(e.length===0)return[];let t=e[0].length,s=new Array(t).fill(0);for(let o of e)for(let n=0;n<t;n++)s[n]+=o[n]??0;for(let o=0;o<t;o++)s[o]/=e.length;return s}throw new Error("Unsupported embedding shape")}var ie=class{constructor(e,t=new Q){this.primary=e;this.fallback=t}primary;fallback;name="resilient";async embed(e){if(this.primary)try{return await this.primary.embed(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embed(e)}async embedMany(e){if(this.primary)try{return await this.primary.embedMany(e)}catch(t){console.warn("[embedding] primary failed, fallback enabled:",t instanceof Error?t.message:t)}return this.fallback.embedMany(e)}};function Nt(r){return`https://api.cloudflare.com/client/v4/accounts/${r}/ai/v1`}function Ot(){let r=we().embedding;if(r.provider==="local")return null;if(r.provider==="huggingface")return!r.apiKey||!r.model?null:new X({token:r.apiKey,model:r.model});if(r.provider==="gemini")return!r.apiKey||!r.model?null:new J({apiKey:r.apiKey,model:r.model,baseUrl:r.baseUrl});let e=r.baseUrl;return r.provider==="cloudflare"&&!e&&r.accountId&&(e=Nt(r.accountId)),e&&r.model?new Z({apiKey:r.apiKey,baseUrl:e,model:r.model}):null}function Pe(){return new ie(Ot())}async function jt(){let r=le(),e=new F().detect(),t=ye({provider:r.storage.provider,huggingface:r.storage.huggingface,localRoot:r.storage.localRoot}),s=new z(t,e.id,e.name,e.remote??e.name),o=new A,n=await new j(s).load(e.id);if(!n)throw new Error("Code graph missing. Run npm run code:index:test first.");o.import(n.symbols,n.edges);let a=new W({projectId:e.id,rootPath:e.rootPath,model:process.env.HF_EMBEDDING_MODEL??"sentence-transformers/all-MiniLM-L6-v2",storage:s,embeddings:Pe(),graph:o}),c=await a.initialize(),l=process.argv.slice(2).join(" ")||"memory storage hugging face",d=await a.search(l,5);console.log({ok:!0,query:l,...c,results:d.map(p=>({score:Number(p.score.toFixed(4)),file:p.chunk.filePath,symbol:p.chunk.symbolName,lines:`${p.chunk.startLine}-${p.chunk.endLine}`,preview:p.chunk.content.replace(/\s+/g," ").slice(0,140)}))})}jt().catch(r=>{console.error(r),process.exit(1)});
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import{existsSync as E}from"node:fs";import{dirname as M,join as O}from"node:path";import{fileURLToPath as k}from"node:url";import{spawnSync as C}from"node:child_process";import{createConnection as T}from"node:net";import{homedir as y}from"node:os";import{join as N}from"node:path";function v(){return process.env.TOOLNET_SERVICE_SOCKET??N(y(),".toolnet","run","service.sock")}function P(e,t={}){let o=t.socketPath??v(),f=t.timeoutMs??1e3;return new Promise((h,c)=>{let n=T({path:o}),a="",u=!1,l=i=>{u||(u=!0,clearTimeout(S),n.destroy(),i())},S=setTimeout(()=>{l(()=>{c(new Error(`ToolNet service timeout after ${f}ms`))})},f);n.setEncoding("utf8"),n.once("connect",()=>{n.write(`${JSON.stringify(e)}
|
|
2
|
+
`)}),n.on("data",i=>{a+=i;let m=a.indexOf(`
|
|
3
|
+
`);if(m===-1)return;let w=a.slice(0,m).trim();l(()=>{try{h(JSON.parse(w))}catch(p){c(p instanceof Error?p:new Error(String(p)))}})}),n.once("error",i=>{l(()=>{c(i)})}),n.once("end",()=>{u||l(()=>{c(new Error("ToolNet service closed without response"))})})})}async function d(e={}){let t=await P({type:"ping"},e);if(!t.ok||t.type!=="ping")throw new Error(t.ok?"Invalid ToolNet service ping response":t.error);return t}var s=process.env.TOOLNET_PM2_NAME??"toolnet-memory-service";function r(e,t=!1){let o=C("pm2",e,{encoding:"utf8",stdio:t?"pipe":"inherit"});if(o.error)throw o.error;return o}function x(){if(r(["--version"],!0).status!==0)throw new Error("PM2 is not installed. Install it with: npm install -g pm2")}function R(){let e=M(k(import.meta.url)),t=O(e,"service.js");if(!E(t))throw new Error(`Production service bundle missing: ${t}`);return t}function $(){return r(["describe",s],!0).status===0}async function g(e){try{let t=await d({timeoutMs:500});if(e){console.log(JSON.stringify({running:!0,...t.stats},null,2));return}console.log(""),console.log("\u25C7 ToolNet Memory Service"),console.log(""),console.log("\u25C6 Daemon"),console.log("\u2502"),console.log("\u251C \u25C6 Status \u2014 running"),console.log(`\u251C \u25C6 PID \u2014 ${t.stats.pid}`),console.log(`\u251C \u25C6 Cache entries \u2014 ${t.stats.cacheEntries}`),console.log(`\u251C \u25C6 Cache hits \u2014 ${t.stats.cacheHits}`),console.log(`\u251C \u25C6 Cache misses \u2014 ${t.stats.cacheMisses}`),console.log("\u2502"),console.log("\u2514 \u25C6 Service healthy"),console.log("")}catch{if(e){console.log(JSON.stringify({running:!1},null,2));return}console.log(""),console.log("\u25C7 ToolNet Memory Service"),console.log(""),console.log("\u25C6 Daemon"),console.log("\u2502"),console.log("\u2514 \u25C7 Status \u2014 stopped"),console.log("")}}async function b(){let e=process.argv[2]??"status",t=process.argv.includes("--json");if(e==="status"){await g(t);return}if(x(),e==="install"){if($()){if(r(["restart",s,"--update-env"],!0).status!==0)throw new Error("PM2 restart failed")}else if(r(["start",R(),"--name",s,"--interpreter",process.execPath,"--time"],!0).status!==0)throw new Error("PM2 install failed");r(["save"],!0),await g(!1);return}if(e==="start"){if(r(["start",s],!0).status!==0)throw new Error("Service is not installed. Run toolnet-memory service:install");return}if(e==="stop"){if(r(["stop",s],!0).status!==0)throw new Error("PM2 stop failed");return}if(e==="restart"){if(r(["restart",s,"--update-env"],!0).status!==0)throw new Error("PM2 restart failed");return}if(e==="remove"){if(r(["delete",s],!0).status!==0)throw new Error("PM2 delete failed");r(["save"],!0);return}throw new Error(`Unknown service command: ${e}`)}b().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exitCode=1});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import{chmodSync as je,existsSync as M,mkdirSync as Pe,unlinkSync as D}from"node:fs";import{createConnection as we,createServer as xe}from"node:net";import{dirname as ke,resolve as Te}from"node:path";import{pathToFileURL as be}from"node:url";import{existsSync as B,readFileSync as H}from"node:fs";import{homedir as J}from"node:os";import{join as V}from"node:path";function L(o){let e=o.trim();return e.length>=2&&e.startsWith('"')&&e.endsWith('"')?(e=e.slice(1,-1),e.replace(/\\n/g,`
|
|
2
|
+
`).replace(/\\r/g,"\r").replace(/\\t/g," ").replace(/\\"/g,'"').replace(/\\\\/g,"\\")):e.length>=2&&e.startsWith("'")&&e.endsWith("'")?e.slice(1,-1):e}function z(){let o=process.env.TOOLNET_GLOBAL_ENV??V(J(),".config","toolnet-memory",".env");if(!B(o))return;let e=H(o,"utf8");for(let t of e.split(/\r?\n/)){let r=t.trim();if(!r||r.startsWith("#"))continue;r.startsWith("export ")&&(r=r.slice(7));let s=r.indexOf("=");if(s<=0)continue;let n=r.slice(0,s).trim();/^[A-Za-z_][A-Za-z0-9_]*$/.test(n)&&process.env[n]===void 0&&(process.env[n]=L(r.slice(s+1)))}}z();function m(o,e){return o===void 0?e:["1","true","yes","on"].includes(o.toLowerCase())}function y(o,e){if(!o)return e;let t=Number(o);return Number.isFinite(t)?t:e}function C(){return{memory:{autoCapture:m(process.env.MEMORY_AUTO_CAPTURE,!0),autoRetrieve:m(process.env.MEMORY_AUTO_RETRIEVE,!0),autoSummarize:m(process.env.MEMORY_AUTO_SUMMARIZE,!0),autoSync:m(process.env.MEMORY_AUTO_SYNC,!0)},retrieval:{maxCandidates:y(process.env.MEMORY_MAX_CANDIDATES,50),rerankTop:y(process.env.MEMORY_RERANK_TOP,10),finalContext:y(process.env.MEMORY_FINAL_CONTEXT,5),tokenBudget:y(process.env.MEMORY_TOKEN_BUDGET,2e3)},storage:{provider:process.env.MEMORY_STORAGE_PROVIDER??"huggingface",r2:{accountId:process.env.R2_ACCOUNT_ID,bucket:process.env.R2_BUCKET,accessKeyId:process.env.R2_ACCESS_KEY_ID,secretAccessKey:process.env.R2_SECRET_ACCESS_KEY},s3:{endpoint:process.env.S3_ENDPOINT,region:process.env.S3_REGION,bucket:process.env.S3_BUCKET,accessKeyId:process.env.S3_ACCESS_KEY_ID,secretAccessKey:process.env.S3_SECRET_ACCESS_KEY,forcePathStyle:m(process.env.S3_FORCE_PATH_STYLE,!1)},huggingface:{namespace:process.env.HF_NAMESPACE,bucket:process.env.HF_BUCKET,accessKeyId:process.env.HF_S3_ACCESS_KEY_ID,secretAccessKey:process.env.HF_S3_SECRET_ACCESS_KEY},localRoot:process.env.MEMORY_LOCAL_STORAGE_PATH},cache:{maxMb:y(process.env.MEMORY_LOCAL_CACHE_MB,200)}}}import{homedir as me}from"node:os";import{join as ye}from"node:path";import{DeleteObjectCommand as F,GetObjectCommand as Y,HeadObjectCommand as G,ListObjectsV2Command as W,PutObjectCommand as q,S3Client as Q}from"@aws-sdk/client-s3";import{getSignedUrl as Z}from"@aws-sdk/s3-request-presigner";var v=class{name="huggingface";client;bucket;constructor(e){this.bucket=e.bucket,this.client=new Q({region:"us-east-1",endpoint:`https://s3.hf.co/${e.namespace}`,forcePathStyle:!0,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let s=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new q({Bucket:this.bucket,Key:e,Body:s,ContentType:r}))}async get(e){let t=await Z(this.client,new Y({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`HF download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new G({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new F({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let s=await this.client.send(new W({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of s.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=s.IsTruncated?s.NextContinuationToken:void 0}while(r);return t}};import{access as O,mkdir as X,readFile as ee,readdir as te,rm as re,stat as E,writeFile as oe}from"node:fs/promises";import{dirname as se,join as ne,relative as A,resolve as ie}from"node:path";var f=class{constructor(e){this.root=e}root;name="local";path(e){let t=e.replace(/^\/+/,"");return ie(this.root,t)}async put(e,t){let r=this.path(e);await X(se(r),{recursive:!0}),await oe(r,t)}async get(e){try{return await ee(this.path(e))}catch(t){if(typeof t=="object"&&t!==null&&"code"in t&&t.code==="ENOENT")return null;throw t}}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await O(this.path(e)),!0}catch{return!1}}async delete(e){await re(this.path(e),{force:!0})}async list(e=""){let t=this.path(e),r=[];try{await O(t)}catch{return r}let s=async i=>{let c=await te(i,{withFileTypes:!0});for(let g of c){let l=ne(i,g.name);if(g.isDirectory()){await s(l);continue}let a=await E(l);r.push({key:A(this.root,l),size:a.size,updatedAt:a.mtime.toISOString()})}},n=await E(t);return n.isDirectory()?await s(t):r.push({key:A(this.root,t),size:n.size,updatedAt:n.mtime.toISOString()}),r}};import{DeleteObjectCommand as ae,GetObjectCommand as ce,HeadObjectCommand as pe,ListObjectsV2Command as ue,PutObjectCommand as le,S3Client as de}from"@aws-sdk/client-s3";import{getSignedUrl as ge}from"@aws-sdk/s3-request-presigner";var h=class{name;client;bucket;constructor(e){this.name=e.name??"s3",this.bucket=e.bucket,this.client=new de({region:e.region??"us-east-1",endpoint:e.endpoint||void 0,forcePathStyle:e.forcePathStyle??!1,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let s=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new le({Bucket:this.bucket,Key:e,Body:s,ContentType:r}))}async get(e){let t=await ge(this.client,new ce({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`${this.name} download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new pe({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new ae({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let s=await this.client.send(new ue({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of s.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=s.IsTruncated?s.NextContinuationToken:void 0}while(r);return t}};function T(o,e){return console.warn(e),new f(o)}function R(o){let e=o.localRoot??ye(me(),".toolnet-memory","storage");if(o.provider==="r2"){let t=o.r2;return t?.accountId&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new h({name:"r2",endpoint:`https://${t.accountId}.r2.cloudflarestorage.com`,region:"auto",bucket:t.bucket,forcePathStyle:!0,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):T(e,"[storage] Cloudflare R2 credentials missing. Using local fallback.")}if(o.provider==="s3"){let t=o.s3;return t?.bucket&&t.accessKeyId&&t.secretAccessKey?new h({name:"s3",endpoint:t.endpoint,region:t.region??"us-east-1",bucket:t.bucket,forcePathStyle:t.forcePathStyle??!1,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):T(e,"[storage] S3 credentials missing. Using local fallback.")}if(o.provider==="huggingface"){let t=o.huggingface;return t?.namespace&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new v({namespace:t.namespace,bucket:t.bucket,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):T(e,"[storage] Hugging Face credentials missing. Using local fallback.")}return new f(e)}var j=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"memories","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));if(!t)return[];let r=JSON.parse(t);if(!Array.isArray(r))throw new Error("Invalid memory snapshot");return r}async save(e,t){let r=JSON.stringify(t,null,2);await this.storage.put(this.key(e),r,"application/json")}};var P=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"graph","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};function fe(o){return new Promise(e=>setTimeout(e,o))}async function N(o,e={}){let t=Math.max(1,e.attempts??3),r=e.baseDelayMs??150,s=e.maxDelayMs??2e3,n;for(let i=1;i<=t;i++)try{return await o()}catch(c){if(n=c,i>=t)break;let g=Math.min(s,r*2**(i-1)),l=Math.floor(Math.random()*Math.max(1,g*.2));await fe(g+l)}throw n}var he=new Set(["put","get","getText","delete","list"]);function I(o,e={}){return new Proxy(o,{get(t,r){let s=Reflect.get(t,r,t);return typeof s!="function"?s:he.has(r)?(...n)=>N(()=>Promise.resolve(s.apply(t,n)),e):s.bind(t)}})}function _(o){let e=o.trim().replace(/\s+/g,"_").replace(/[^A-Za-z0-9._-]/g,"_").replace(/_+/g,"_").replace(/^\.+|\.+$/g,"").slice(0,100);if(!e||e==="."||e==="..")throw new Error("Invalid project storage folder");return e}function K(o){let e=o.replace(/^\/+/,"");if(e.startsWith("memories/"))return"memory/records/"+e.slice(9);if(e.startsWith("vectors/"))return"memory/vectors/"+e.slice(8);if(e.startsWith("graph/"))return"code/graph/"+e.slice(6);let t=e.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);if(t)return`${t[1]}memory/records/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/),t)return`${t[1]}memory/vectors/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/),t)return`${t[1]}code/graph/${t[2]}`;let r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);if(r)return`${r[1]}memory/records/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/),r)return`${r[1]}memory/vectors/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/),r)return`${r[1]}code/graph/${r[2]}`;let s=e.match(/^(projects\/[^/]+\/)memories\/(.+)$/);return s?`${s[1]}memory/records/${s[2]}`:(s=e.match(/^(projects\/[^/]+\/)vectors\/(.+)$/),s?`${s[1]}memory/vectors/${s[2]}`:(s=e.match(/^(projects\/[^/]+\/)graph\/(.+)$/),s?`${s[1]}code/graph/${s[2]}`:e))}var w=class{constructor(e,t,r,s){this.provider=e;this.name=e.name,this.projectId=t,this.projectName=r,this.folder=_(s??r),this.sourcePrefix=`projects/${t}`,this.targetPrefix=`projects/${this.folder}`}provider;name;folder;sourcePrefix;targetPrefix;projectId;projectName;registryPromise;async registerProject(){let e=`${this.targetPrefix}/project.json`,t=new Date().toISOString(),r=t,s=await this.provider.getText(e);if(s){let i;try{i=JSON.parse(s)}catch(c){throw new Error(`Invalid remote ToolNet project manifest at ${e}: ${c instanceof Error?c.message:String(c)}`)}if(typeof i.id=="string"&&i.id!==this.projectId)throw new Error(["ToolNet remote project namespace collision.",`Remote folder: ${this.targetPrefix}`,`Existing project id: ${i.id}`,`Current project id: ${this.projectId}`,"Refusing to mix data from two projects."].join(" "));typeof i.createdAt=="string"&&(r=i.createdAt)}let n={version:1,id:this.projectId,name:this.projectName,remote:this.folder,createdAt:r,updatedAt:t};await this.provider.put(e,JSON.stringify(n,null,2)+`
|
|
3
|
+
`,"application/json")}async ensureRegistered(){return this.registryPromise||(this.registryPromise=this.registerProject()),this.registryPromise}key(e){if(e=K(e),e===this.sourcePrefix)return this.targetPrefix;if(e.startsWith(`${this.sourcePrefix}/`))return this.targetPrefix+e.slice(this.sourcePrefix.length);if(e===this.targetPrefix||e.startsWith(`${this.targetPrefix}/`))return e;if(e.startsWith("projects/"))throw new Error(["Cross-project storage access denied.",`Current project: ${this.targetPrefix}`,`Requested key: ${e}`].join(" "));return e}async put(e,t,r){return await this.ensureRegistered(),this.provider.put(this.key(e),t,r)}async get(e){return await this.ensureRegistered(),this.provider.get(this.key(e))}async getText(e){return await this.ensureRegistered(),this.provider.getText(this.key(e))}async delete(e){return await this.ensureRegistered(),this.provider.delete(this.key(e))}async exists(e){return await this.ensureRegistered(),this.provider.exists(this.key(e))}async list(e){return await this.ensureRegistered(),this.provider.list(this.key(e))}};import{homedir as Se}from"node:os";import{join as ve}from"node:path";function $(){return process.env.TOOLNET_SERVICE_SOCKET??ve(Se(),".toolnet","run","service.sock")}function b(o){return o instanceof Error?o.message:String(o)}function Ce(o){return`${o.id}:${o.remote??o.name}`}async function Oe(o){let e=C(),t=R({provider:e.storage.provider,r2:e.storage.r2,s3:e.storage.s3,huggingface:e.storage.huggingface,localRoot:e.storage.localRoot}),r=I(t,{attempts:Number(process.env.TOOLNET_STORAGE_RETRIES??3)}),s=new w(r,o.id,o.name,o.remote??o.name),[n,i]=await Promise.all([new j(s).load(o.id),new P(s).load(o.id)]);return{memory:n,graph:i}}function Ee(o,e=200){return new Promise(t=>{let r=we({path:o}),s=!1,n=c=>{s||(s=!0,clearTimeout(i),r.destroy(),t(c))},i=setTimeout(()=>{n(!1)},e);r.once("connect",()=>{n(!0)}),r.once("error",()=>{n(!1)})})}async function Ae(o={}){let e=o.socketPath??$(),t=Math.max(1e3,o.cacheTtlMs??Number(process.env.TOOLNET_SERVICE_CACHE_TTL_MS??3e4)),r=o.loader??Oe,s=new Map,n=new Date().toISOString(),i={requests:0,cacheHits:0,cacheMisses:0},c=()=>({pid:process.pid,startedAt:n,requests:i.requests,cacheHits:i.cacheHits,cacheMisses:i.cacheMisses,cacheEntries:s.size}),g=async a=>{if(i.requests+=1,a.type==="ping")return{ok:!0,type:"ping",stats:c()};let p=Ce(a.project);if(a.type==="invalidate")return{ok:!0,type:"invalidate",removed:s.delete(p)};let d=s.get(p);if(d&&d.expiresAt>Date.now())return i.cacheHits+=1,{ok:!0,type:"hydrate",cacheHit:!0,loadedAt:d.loadedAt,memory:d.data.memory,graph:d.data.graph};i.cacheMisses+=1;try{let u=await r(a.project),S=new Date().toISOString();return s.set(p,{expiresAt:Date.now()+t,loadedAt:S,data:u}),{ok:!0,type:"hydrate",cacheHit:!1,loadedAt:S,memory:u.memory,graph:u.graph}}catch(u){return{ok:!1,error:b(u)}}};if(Pe(ke(e),{recursive:!0,mode:448}),M(e)){if(await Ee(e))throw new Error(`ToolNet service already running at ${e}`);D(e)}let l=xe(a=>{a.setEncoding("utf8");let p="";a.on("data",d=>{p+=d;let u=p.indexOf(`
|
|
4
|
+
`);if(u===-1)return;let S=p.slice(0,u).trim();p=p.slice(u+1),(async()=>{try{let x=JSON.parse(S),k=await g(x);a.end(`${JSON.stringify(k)}
|
|
5
|
+
`)}catch(x){let k={ok:!1,error:b(x)};a.end(`${JSON.stringify(k)}
|
|
6
|
+
`)}})()})});return await new Promise((a,p)=>{let d=u=>{p(u)};l.once("error",d),l.listen(e,()=>{l.off("error",d),a()})}),je(e,384),{socketPath:e,stats:c,close:async()=>{await new Promise(a=>{l.close(()=>{a()})}),M(e)&&D(e)}}}async function Re(){let o=await Ae();console.log(`[toolnet-memory] service ready: ${o.socketPath}`);let e=()=>{o.close().finally(()=>process.exit(0))};process.once("SIGINT",e),process.once("SIGTERM",e)}var U=process.argv[1];U&&import.meta.url===be(Te(U)).href&&Re().catch(o=>{console.error(`[toolnet-memory] service failed: ${b(o)}`),process.exit(1)});export{Oe as loadProjectForService,Ae as startToolNetService};
|