toolnet-memory 0.3.1 → 0.3.3

package/bundle/service-cli.js

@@ -0,0 +1,3 @@
+import{existsSync as E}from"node:fs";import{dirname as M,join as O}from"node:path";import{fileURLToPath as k}from"node:url";import{spawnSync as C}from"node:child_process";import{createConnection as T}from"node:net";import{homedir as y}from"node:os";import{join as N}from"node:path";function v(){return process.env.TOOLNET_SERVICE_SOCKET??N(y(),".toolnet","run","service.sock")}function P(e,t={}){let o=t.socketPath??v(),f=t.timeoutMs??1e3;return new Promise((S,c)=>{let n=T({path:o}),a="",u=!1,l=i=>{u||(u=!0,clearTimeout(h),n.destroy(),i())},h=setTimeout(()=>{l(()=>{c(new Error(`ToolNet service timeout after ${f}ms`))})},f);n.setEncoding("utf8"),n.once("connect",()=>{n.write(`${JSON.stringify(e)}
+`)}),n.on("data",i=>{a+=i;let m=a.indexOf(`
+`);if(m===-1)return;let w=a.slice(0,m).trim();l(()=>{try{S(JSON.parse(w))}catch(p){c(p instanceof Error?p:new Error(String(p)))}})}),n.once("error",i=>{l(()=>{c(i)})}),n.once("end",()=>{u||l(()=>{c(new Error("ToolNet service closed without response"))})})})}async function d(e={}){let t=await P({type:"ping"},e);if(!t.ok||t.type!=="ping")throw new Error(t.ok?"Invalid ToolNet service ping response":t.error);return t}var s=process.env.TOOLNET_PM2_NAME??"toolnet-memory-service";function r(e,t=!1){let o=C("pm2",e,{encoding:"utf8",stdio:t?"pipe":"inherit"});if(o.error)throw o.error;return o}function x(){if(r(["--version"],!0).status!==0)throw new Error("PM2 is not installed. Install it with: npm install -g pm2")}function R(){let e=M(k(import.meta.url)),t=O(e,"service.js");if(!E(t))throw new Error(`Production service bundle missing: ${t}`);return t}function $(){return r(["describe",s],!0).status===0}async function g(e){try{let t=await d({timeoutMs:500});if(e){console.log(JSON.stringify({running:!0,...t.stats},null,2));return}console.log(""),console.log("ToolNet Memory Service"),console.log("======================"),console.log(""),console.log("State: running"),console.log(`PID: ${t.stats.pid}`),console.log(`Cache entries: ${t.stats.cacheEntries}`),console.log(`Cache hits: ${t.stats.cacheHits}`),console.log(`Cache misses: ${t.stats.cacheMisses}`),console.log("")}catch{if(e){console.log(JSON.stringify({running:!1},null,2));return}console.log(""),console.log("ToolNet Memory Service"),console.log("======================"),console.log(""),console.log("State: stopped"),console.log("")}}async function b(){let e=process.argv[2]??"status",t=process.argv.includes("--json");if(e==="status"){await g(t);return}if(x(),e==="install"){if($()){if(r(["restart",s,"--update-env"]).status!==0)throw new Error("PM2 restart failed")}else if(r(["start",R(),"--name",s,"--interpreter",process.execPath,"--time"]).status!==0)throw new Error("PM2 install failed");r(["save"]),await g(!1);return}if(e==="start"){if(r(["start",s]).status!==0)throw new Error("Service is not installed. Run toolnet-memory service:install");return}if(e==="stop"){if(r(["stop",s]).status!==0)throw new Error("PM2 stop failed");return}if(e==="restart"){if(r(["restart",s,"--update-env"]).status!==0)throw new Error("PM2 restart failed");return}if(e==="remove"){if(r(["delete",s]).status!==0)throw new Error("PM2 delete failed");r(["save"]);return}throw new Error(`Unknown service command: ${e}`)}b().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exitCode=1});

package/bundle/service.js

@@ -0,0 +1,6 @@
+import{chmodSync as je,existsSync as M,mkdirSync as Pe,unlinkSync as D}from"node:fs";import{createConnection as we,createServer as xe}from"node:net";import{dirname as ke,resolve as Te}from"node:path";import{pathToFileURL as be}from"node:url";import{existsSync as B,readFileSync as H}from"node:fs";import{homedir as J}from"node:os";import{join as V}from"node:path";function L(o){let e=o.trim();return e.length>=2&&e.startsWith('"')&&e.endsWith('"')?(e=e.slice(1,-1),e.replace(/\\n/g,`
+`).replace(/\\r/g,"\r").replace(/\\t/g,"	").replace(/\\"/g,'"').replace(/\\\\/g,"\\")):e.length>=2&&e.startsWith("'")&&e.endsWith("'")?e.slice(1,-1):e}function z(){let o=process.env.TOOLNET_GLOBAL_ENV??V(J(),".config","toolnet-memory",".env");if(!B(o))return;let e=H(o,"utf8");for(let t of e.split(/\r?\n/)){let r=t.trim();if(!r||r.startsWith("#"))continue;r.startsWith("export ")&&(r=r.slice(7));let s=r.indexOf("=");if(s<=0)continue;let n=r.slice(0,s).trim();/^[A-Za-z_][A-Za-z0-9_]*$/.test(n)&&process.env[n]===void 0&&(process.env[n]=L(r.slice(s+1)))}}z();function m(o,e){return o===void 0?e:["1","true","yes","on"].includes(o.toLowerCase())}function y(o,e){if(!o)return e;let t=Number(o);return Number.isFinite(t)?t:e}function C(){return{memory:{autoCapture:m(process.env.MEMORY_AUTO_CAPTURE,!0),autoRetrieve:m(process.env.MEMORY_AUTO_RETRIEVE,!0),autoSummarize:m(process.env.MEMORY_AUTO_SUMMARIZE,!0),autoSync:m(process.env.MEMORY_AUTO_SYNC,!0)},retrieval:{maxCandidates:y(process.env.MEMORY_MAX_CANDIDATES,50),rerankTop:y(process.env.MEMORY_RERANK_TOP,10),finalContext:y(process.env.MEMORY_FINAL_CONTEXT,5),tokenBudget:y(process.env.MEMORY_TOKEN_BUDGET,2e3)},storage:{provider:process.env.MEMORY_STORAGE_PROVIDER??"huggingface",r2:{accountId:process.env.R2_ACCOUNT_ID,bucket:process.env.R2_BUCKET,accessKeyId:process.env.R2_ACCESS_KEY_ID,secretAccessKey:process.env.R2_SECRET_ACCESS_KEY},s3:{endpoint:process.env.S3_ENDPOINT,region:process.env.S3_REGION,bucket:process.env.S3_BUCKET,accessKeyId:process.env.S3_ACCESS_KEY_ID,secretAccessKey:process.env.S3_SECRET_ACCESS_KEY,forcePathStyle:m(process.env.S3_FORCE_PATH_STYLE,!1)},huggingface:{namespace:process.env.HF_NAMESPACE,bucket:process.env.HF_BUCKET,accessKeyId:process.env.HF_S3_ACCESS_KEY_ID,secretAccessKey:process.env.HF_S3_SECRET_ACCESS_KEY},localRoot:process.env.MEMORY_LOCAL_STORAGE_PATH},cache:{maxMb:y(process.env.MEMORY_LOCAL_CACHE_MB,200)}}}import{homedir as me}from"node:os";import{join as ye}from"node:path";import{DeleteObjectCommand as F,GetObjectCommand as Y,HeadObjectCommand as G,ListObjectsV2Command as W,PutObjectCommand as q,S3Client as Q}from"@aws-sdk/client-s3";import{getSignedUrl as Z}from"@aws-sdk/s3-request-presigner";var v=class{name="huggingface";client;bucket;constructor(e){this.bucket=e.bucket,this.client=new Q({region:"us-east-1",endpoint:`https://s3.hf.co/${e.namespace}`,forcePathStyle:!0,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let s=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new q({Bucket:this.bucket,Key:e,Body:s,ContentType:r}))}async get(e){let t=await Z(this.client,new Y({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`HF download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new G({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new F({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let s=await this.client.send(new W({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of s.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=s.IsTruncated?s.NextContinuationToken:void 0}while(r);return t}};import{access as O,mkdir as X,readFile as ee,readdir as te,rm as re,stat as E,writeFile as oe}from"node:fs/promises";import{dirname as se,join as ne,relative as A,resolve as ie}from"node:path";var f=class{constructor(e){this.root=e}root;name="local";path(e){let t=e.replace(/^\/+/,"");return ie(this.root,t)}async put(e,t){let r=this.path(e);await X(se(r),{recursive:!0}),await oe(r,t)}async get(e){try{return await ee(this.path(e))}catch(t){if(typeof t=="object"&&t!==null&&"code"in t&&t.code==="ENOENT")return null;throw t}}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await O(this.path(e)),!0}catch{return!1}}async delete(e){await re(this.path(e),{force:!0})}async list(e=""){let t=this.path(e),r=[];try{await O(t)}catch{return r}let s=async i=>{let c=await te(i,{withFileTypes:!0});for(let g of c){let l=ne(i,g.name);if(g.isDirectory()){await s(l);continue}let a=await E(l);r.push({key:A(this.root,l),size:a.size,updatedAt:a.mtime.toISOString()})}},n=await E(t);return n.isDirectory()?await s(t):r.push({key:A(this.root,t),size:n.size,updatedAt:n.mtime.toISOString()}),r}};import{DeleteObjectCommand as ae,GetObjectCommand as ce,HeadObjectCommand as pe,ListObjectsV2Command as ue,PutObjectCommand as le,S3Client as de}from"@aws-sdk/client-s3";import{getSignedUrl as ge}from"@aws-sdk/s3-request-presigner";var h=class{name;client;bucket;constructor(e){this.name=e.name??"s3",this.bucket=e.bucket,this.client=new de({region:e.region??"us-east-1",endpoint:e.endpoint||void 0,forcePathStyle:e.forcePathStyle??!1,requestChecksumCalculation:"WHEN_REQUIRED",responseChecksumValidation:"WHEN_REQUIRED",credentials:{accessKeyId:e.accessKeyId,secretAccessKey:e.secretAccessKey}})}async put(e,t,r="application/octet-stream"){let s=typeof t=="string"?Buffer.from(t,"utf8"):t;await this.client.send(new le({Bucket:this.bucket,Key:e,Body:s,ContentType:r}))}async get(e){let t=await ge(this.client,new ce({Bucket:this.bucket,Key:e}),{expiresIn:60}),r=await fetch(t,{redirect:"follow"});if(r.status===404)return null;if(!r.ok)throw new Error(`${this.name} download failed: ${r.status} ${r.statusText}`);return new Uint8Array(await r.arrayBuffer())}async getText(e){let t=await this.get(e);return t?Buffer.from(t).toString("utf8"):null}async exists(e){try{return await this.client.send(new pe({Bucket:this.bucket,Key:e})),!0}catch(t){if(typeof t=="object"&&t!==null&&"$metadata"in t&&t.$metadata?.httpStatusCode===404)return!1;throw t}}async delete(e){await this.client.send(new ae({Bucket:this.bucket,Key:e}))}async list(e=""){let t=[],r;do{let s=await this.client.send(new ue({Bucket:this.bucket,Prefix:e||void 0,ContinuationToken:r}));for(let n of s.Contents??[])n.Key&&t.push({key:n.Key,size:n.Size,updatedAt:n.LastModified?.toISOString()});r=s.IsTruncated?s.NextContinuationToken:void 0}while(r);return t}};function T(o,e){return console.warn(e),new f(o)}function R(o){let e=o.localRoot??ye(me(),".toolnet-memory","storage");if(o.provider==="r2"){let t=o.r2;return t?.accountId&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new h({name:"r2",endpoint:`https://${t.accountId}.r2.cloudflarestorage.com`,region:"auto",bucket:t.bucket,forcePathStyle:!0,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):T(e,"[storage] Cloudflare R2 credentials missing. Using local fallback.")}if(o.provider==="s3"){let t=o.s3;return t?.bucket&&t.accessKeyId&&t.secretAccessKey?new h({name:"s3",endpoint:t.endpoint,region:t.region??"us-east-1",bucket:t.bucket,forcePathStyle:t.forcePathStyle??!1,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):T(e,"[storage] S3 credentials missing. Using local fallback.")}if(o.provider==="huggingface"){let t=o.huggingface;return t?.namespace&&t.bucket&&t.accessKeyId&&t.secretAccessKey?new v({namespace:t.namespace,bucket:t.bucket,accessKeyId:t.accessKeyId,secretAccessKey:t.secretAccessKey}):T(e,"[storage] Hugging Face credentials missing. Using local fallback.")}return new f(e)}var j=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"memories","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));if(!t)return[];let r=JSON.parse(t);if(!Array.isArray(r))throw new Error("Invalid memory snapshot");return r}async save(e,t){let r=JSON.stringify(t,null,2);await this.storage.put(this.key(e),r,"application/json")}};var P=class{constructor(e){this.storage=e}storage;key(e){return["projects",e,"graph","current.json"].join("/")}async load(e){let t=await this.storage.getText(this.key(e));return t?JSON.parse(t):null}async save(e){await this.storage.put(this.key(e.projectId),JSON.stringify(e),"application/json")}};function fe(o){return new Promise(e=>setTimeout(e,o))}async function N(o,e={}){let t=Math.max(1,e.attempts??3),r=e.baseDelayMs??150,s=e.maxDelayMs??2e3,n;for(let i=1;i<=t;i++)try{return await o()}catch(c){if(n=c,i>=t)break;let g=Math.min(s,r*2**(i-1)),l=Math.floor(Math.random()*Math.max(1,g*.2));await fe(g+l)}throw n}var he=new Set(["put","get","getText","delete","list"]);function I(o,e={}){return new Proxy(o,{get(t,r){let s=Reflect.get(t,r,t);return typeof s!="function"?s:he.has(r)?(...n)=>N(()=>Promise.resolve(s.apply(t,n)),e):s.bind(t)}})}function _(o){let e=o.trim().replace(/\s+/g,"_").replace(/[^A-Za-z0-9._-]/g,"_").replace(/_+/g,"_").replace(/^\.+|\.+$/g,"").slice(0,100);if(!e||e==="."||e==="..")throw new Error("Invalid project storage folder");return e}function K(o){let e=o.replace(/^\/+/,"");if(e.startsWith("memories/"))return"memory/records/"+e.slice(9);if(e.startsWith("vectors/"))return"memory/vectors/"+e.slice(8);if(e.startsWith("graph/"))return"code/graph/"+e.slice(6);let t=e.match(/^(snapshots\/[^/]+\/)memories\/(.+)$/);if(t)return`${t[1]}memory/records/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)vectors\/(.+)$/),t)return`${t[1]}memory/vectors/${t[2]}`;if(t=e.match(/^(snapshots\/[^/]+\/)graph\/(.+)$/),t)return`${t[1]}code/graph/${t[2]}`;let r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)memories\/(.+)$/);if(r)return`${r[1]}memory/records/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)vectors\/(.+)$/),r)return`${r[1]}memory/vectors/${r[2]}`;if(r=e.match(/^(projects\/[^/]+\/snapshots\/[^/]+\/)graph\/(.+)$/),r)return`${r[1]}code/graph/${r[2]}`;let s=e.match(/^(projects\/[^/]+\/)memories\/(.+)$/);return s?`${s[1]}memory/records/${s[2]}`:(s=e.match(/^(projects\/[^/]+\/)vectors\/(.+)$/),s?`${s[1]}memory/vectors/${s[2]}`:(s=e.match(/^(projects\/[^/]+\/)graph\/(.+)$/),s?`${s[1]}code/graph/${s[2]}`:e))}var w=class{constructor(e,t,r,s){this.provider=e;this.name=e.name,this.projectId=t,this.projectName=r,this.folder=_(s??r),this.sourcePrefix=`projects/${t}`,this.targetPrefix=`projects/${this.folder}`}provider;name;folder;sourcePrefix;targetPrefix;projectId;projectName;registryPromise;async registerProject(){let e=`${this.targetPrefix}/project.json`,t=new Date().toISOString(),r=t,s=await this.provider.getText(e);if(s){let i;try{i=JSON.parse(s)}catch(c){throw new Error(`Invalid remote ToolNet project manifest at ${e}: ${c instanceof Error?c.message:String(c)}`)}if(typeof i.id=="string"&&i.id!==this.projectId)throw new Error(["ToolNet remote project namespace collision.",`Remote folder: ${this.targetPrefix}`,`Existing project id: ${i.id}`,`Current project id: ${this.projectId}`,"Refusing to mix data from two projects."].join(" "));typeof i.createdAt=="string"&&(r=i.createdAt)}let n={version:1,id:this.projectId,name:this.projectName,remote:this.folder,createdAt:r,updatedAt:t};await this.provider.put(e,JSON.stringify(n,null,2)+`
+`,"application/json")}async ensureRegistered(){return this.registryPromise||(this.registryPromise=this.registerProject()),this.registryPromise}key(e){if(e=K(e),e===this.sourcePrefix)return this.targetPrefix;if(e.startsWith(`${this.sourcePrefix}/`))return this.targetPrefix+e.slice(this.sourcePrefix.length);if(e===this.targetPrefix||e.startsWith(`${this.targetPrefix}/`))return e;if(e.startsWith("projects/"))throw new Error(["Cross-project storage access denied.",`Current project: ${this.targetPrefix}`,`Requested key: ${e}`].join(" "));return e}async put(e,t,r){return await this.ensureRegistered(),this.provider.put(this.key(e),t,r)}async get(e){return await this.ensureRegistered(),this.provider.get(this.key(e))}async getText(e){return await this.ensureRegistered(),this.provider.getText(this.key(e))}async delete(e){return await this.ensureRegistered(),this.provider.delete(this.key(e))}async exists(e){return await this.ensureRegistered(),this.provider.exists(this.key(e))}async list(e){return await this.ensureRegistered(),this.provider.list(this.key(e))}};import{homedir as Se}from"node:os";import{join as ve}from"node:path";function $(){return process.env.TOOLNET_SERVICE_SOCKET??ve(Se(),".toolnet","run","service.sock")}function b(o){return o instanceof Error?o.message:String(o)}function Ce(o){return`${o.id}:${o.remote??o.name}`}async function Oe(o){let e=C(),t=R({provider:e.storage.provider,r2:e.storage.r2,s3:e.storage.s3,huggingface:e.storage.huggingface,localRoot:e.storage.localRoot}),r=I(t,{attempts:Number(process.env.TOOLNET_STORAGE_RETRIES??3)}),s=new w(r,o.id,o.name,o.remote??o.name),[n,i]=await Promise.all([new j(s).load(o.id),new P(s).load(o.id)]);return{memory:n,graph:i}}function Ee(o,e=200){return new Promise(t=>{let r=we({path:o}),s=!1,n=c=>{s||(s=!0,clearTimeout(i),r.destroy(),t(c))},i=setTimeout(()=>{n(!1)},e);r.once("connect",()=>{n(!0)}),r.once("error",()=>{n(!1)})})}async function Ae(o={}){let e=o.socketPath??$(),t=Math.max(1e3,o.cacheTtlMs??Number(process.env.TOOLNET_SERVICE_CACHE_TTL_MS??3e4)),r=o.loader??Oe,s=new Map,n=new Date().toISOString(),i={requests:0,cacheHits:0,cacheMisses:0},c=()=>({pid:process.pid,startedAt:n,requests:i.requests,cacheHits:i.cacheHits,cacheMisses:i.cacheMisses,cacheEntries:s.size}),g=async a=>{if(i.requests+=1,a.type==="ping")return{ok:!0,type:"ping",stats:c()};let p=Ce(a.project);if(a.type==="invalidate")return{ok:!0,type:"invalidate",removed:s.delete(p)};let d=s.get(p);if(d&&d.expiresAt>Date.now())return i.cacheHits+=1,{ok:!0,type:"hydrate",cacheHit:!0,loadedAt:d.loadedAt,memory:d.data.memory,graph:d.data.graph};i.cacheMisses+=1;try{let u=await r(a.project),S=new Date().toISOString();return s.set(p,{expiresAt:Date.now()+t,loadedAt:S,data:u}),{ok:!0,type:"hydrate",cacheHit:!1,loadedAt:S,memory:u.memory,graph:u.graph}}catch(u){return{ok:!1,error:b(u)}}};if(Pe(ke(e),{recursive:!0,mode:448}),M(e)){if(await Ee(e))throw new Error(`ToolNet service already running at ${e}`);D(e)}let l=xe(a=>{a.setEncoding("utf8");let p="";a.on("data",d=>{p+=d;let u=p.indexOf(`
+`);if(u===-1)return;let S=p.slice(0,u).trim();p=p.slice(u+1),(async()=>{try{let x=JSON.parse(S),k=await g(x);a.end(`${JSON.stringify(k)}
+`)}catch(x){let k={ok:!1,error:b(x)};a.end(`${JSON.stringify(k)}
+`)}})()})});return await new Promise((a,p)=>{let d=u=>{p(u)};l.once("error",d),l.listen(e,()=>{l.off("error",d),a()})}),je(e,384),{socketPath:e,stats:c,close:async()=>{await new Promise(a=>{l.close(()=>{a()})}),M(e)&&D(e)}}}async function Re(){let o=await Ae();console.log(`[toolnet-memory] service ready: ${o.socketPath}`);let e=()=>{o.close().finally(()=>process.exit(0))};process.once("SIGINT",e),process.once("SIGTERM",e)}var U=process.argv[1];U&&import.meta.url===be(Te(U)).href&&Re().catch(o=>{console.error(`[toolnet-memory] service failed: ${b(o)}`),process.exit(1)});export{Oe as loadProjectForService,Ae as startToolNetService};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "toolnet-memory",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "description": "Persistent project memory, work continuity, and code intelligence for AI coding agents.",
   "type": "module",
   "license": "MIT",