sentinelayer-cli 0.46.5 → 0.48.0

package/src/engram/index.js

@@ -7,11 +7,21 @@
  * store adapter (see the CLI wiring in src/mcp/session-stdio-server.js).
  */
 
-export { createStore, itemId } from "./store.js";
+export { createStore, itemId, LegacyStorageError } from "./store.js";
+export {
+  compileGraph,
+  compileCandidate,
+  CompileGraphError,
+  EW05_PREDICATES,
+  EW05_PREDICATE_SET,
+  ENTITY_KIND,
+  GENERATION_STATUS,
+} from "./compile-graph.js";
 export {
   parseNamespace,
   authorize,
   createLocalConsent,
+  createScopedConsent,
   AccessDeniedError,
   ACTION_SCOPES,
 } from "./namespace.js";
@@ -28,8 +38,55 @@ export {
 export { createGovernance } from "./governance.js";
 export { createMemoryTools } from "./tools.js";
 export { runEngramSla } from "./sla.js";
+export * from "./contracts/index.js";
 export { ingestDocumentIntoNamespace } from "./ingest-document.js";
+export { captureSourceBytes, SourceIntakeError, MAX_SOURCE_BYTES } from "./source-intake.js";
+export { createSafeReader, SafeReaderError } from "./safe-reader.js";
+export { extractFromHtml } from "./extract-html.js";
+export { ingestSite } from "./ingest-site.js";
+export {
+  createGraphStore,
+  GraphValidationError,
+  VersionNotFoundError,
+  IntegrityError,
+} from "./graph-store.js";
+export {
+  buildSiteRetrievalIndex,
+  retrieveSiteEntities,
+  retrieveFromGraphStore,
+  RETRIEVE_SITE_INDEX_VERSION,
+  REQUIRED_EMBEDDER_DIM,
+  SITE_RETRIEVAL_DEFAULTS,
+} from "./retrieve-site.js";
+
 
+
+export {
+  FRESHNESS_CLASSES,
+  classifyFreshness,
+  isValidFreshnessClass,
+  createFreshnessMarker,
+  isFreshnessStale,
+  FreshnessClassError,
+  SOURCE_STATUSES,
+  createSourceVersion,
+  resolveAuthoritativeSource,
+  SourceVersionError,
+  mustRevalidate,
+  RevalidationRequiredError,
+  disabledByOutage,
+  SourceOutageError,
+  createCommerceContext,
+  CommerceContextError,
+} from "./live-state.js";
+export {
+  validateAdapterShape,
+  hydrate,
+  revalidateForAction,
+  LiveStateError,
+  VOLATILE_FIELD_NAMES,
+  isVolatileField,
+} from "./live-state.js";
 import { createStore } from "./store.js";
 import { createLocalConsent } from "./namespace.js";
 import { createGovernance } from "./governance.js";
@@ -38,6 +95,8 @@ import { ingestDocumentIntoNamespace } from "./ingest-document.js";
 import { createMemoryTools } from "./tools.js";
 import { createEmbedder } from "../session/recall/embedder.js";
 
+const DEFAULT_CONSENT = Symbol("default consent");
+
 /**
  * Convenience factory: wire a default CLI-local MaaS instance (permissive-local
  * consent, stub governance, hash-projection embedder) over a store. Callers
@@ -52,22 +111,47 @@ import { createEmbedder } from "../session/recall/embedder.js";
  * @param {object} [options.renderer]
  * @returns {{tools: object, store: object}}
  */
-export function createMemoryService({
-  storeRoot,
-  adapters = {},
-  embedder = createEmbedder(),
-  consent = createLocalConsent(),
-  governance = createGovernance(),
-  renderer = null,
-  isAuthorizedForSealed,
-} = {}) {
-  const store = createStore({ storeRoot, adapters });
+export function createMemoryService(options = {}) {
+  const {
+    storeRoot,
+    adapters = {},
+    embedder = createEmbedder(),
+    consent: rawConsent = DEFAULT_CONSENT,
+    governance = createGovernance(),
+    renderer = null,
+    isAuthorizedForSealed,
+  } = options;
+
+  // Detect whether the caller injected a consent.  When options.consent is
+  // undefined the rawConsent is the DEFAULT_CONSENT sentinel; any other
+  // value (including createLocalConsent() passed explicitly) means the
+  // caller injected one.
+  const callerInjectedConsent = rawConsent !== DEFAULT_CONSENT;
+  const consent = callerInjectedConsent ? rawConsent : createLocalConsent();
+
+  // Default local consent: use "first-claim" to migrate legacy dirs
+  // safely (one user's own namespaces, no cross-tenant risk).
+  // Injected (non-default) consent: "refuse" — never trust legacy dirs
+  // when a scoped or hosted consent is in play.
+  const legacyLossyDirs = callerInjectedConsent ? "refuse" : "first-claim";
+
+  const store = createStore({ storeRoot, adapters, legacyLossyDirs });
   // Wired by DEFAULT, not opt-in: a readiness gate nobody injects is a comment.
-  const buildState = createBuildState({ storeRoot });
+  const buildState = createBuildState({ storeRoot, legacyLossyDirs });
   const tools = createMemoryTools({ store, consent, governance, embedder, renderer, isAuthorizedForSealed, buildState });
   // Exposed on the service so the lifecycle cannot be bypassed by a caller who has a
   // store but forgets the buildState -- the deps are bound here, once.
-  const ingestDocument = (options) =>
-    ingestDocumentIntoNamespace({ ...options, deps: { store, buildState } });
+  const ingestDocument = (opts) =>
+    ingestDocumentIntoNamespace({ ...opts, deps: { store, buildState } });
   return { tools, store, buildState, ingestDocument };
 }
+
+// Capability Index (EW08)
+export {
+  buildCapabilityIndex,
+  verifyCapabilityIndex,
+  toMcpTool,
+  renderSkillText,
+  checkRevocation,
+  findSkill,
+} from "./capability-index.js";

package/src/engram/ingest-site.js

@@ -0,0 +1,132 @@
+/**
+ * ENGRAM §EW04 — Site-to-memory ingestion pipeline.
+ *
+ * Wires: source-intake (authorization) → safe-reader (SSRF-safe fetch) →
+ * extract-html (byte-offset facts) → graph-store (immutable snapshot).
+ *
+ * An unchanged page re-ingests as `reused: true`.
+ * An unavailable source is recorded as unknown, never as empty.
+ *
+ * Library only. Node builtins + existing zod contracts. No new dependency.
+ */
+
+import { captureSourceBytes } from "./source-intake.js";
+import { extractFromHtml } from "./extract-html.js";
+import { parseNamespace } from "./namespace.js";
+
+/**
+ * Ingest a site from an approved source through to the graph store.
+ *
+ * @param {object} options
+ * @param {string} options.sourceId
+ * @param {string} options.tenantScope
+ * @param {Map} options.sources — the approved source registry
+ * @param {Set<string>} options.allowedOrigins
+ * @param {function} options.authorize — tenant policy function
+ * @param {object} options.reader — a safe-reader { read } instance
+ * @param {object} options.graphStore — EW03 graph store (with writeSourceSnapshot)
+ * @param {object} options.caller — caller identity for graph store
+ * @param {string} options.namespace — namespace for graph store
+ * @param {number} [options.maxBytes] — byte cap
+ * @returns {Promise<{snapshot: object, facts: object, reused: boolean}>}
+ */
+export async function ingestSite({
+  sourceId,
+  tenantScope,
+  sources,
+  allowedOrigins,
+  authorize,
+  reader,
+  graphStore,
+  caller,
+  namespace,
+  maxBytes,
+}) {
+  // 1. source-intake: authorize + validate source record (does NOT read bytes)
+  //    We use captureSourceBytes as an authorization gate; we supply our own read function
+  //    that calls safe-reader, but captureSourceBytes handles all the policy checks.
+
+  let captured;
+  try {
+    captured = await captureSourceBytes({
+      sourceId,
+      tenantScope,
+      sources,
+      allowedOrigins,
+      authorize,
+      read: async ({ url, version: approvedVersion }) => {
+        // safe-reader fetch
+        const result = await reader.read(url, { approvedOrigins: allowedOrigins });
+
+        // Pass the reader's ACTUAL version (sha256 of bytes) so source-intake
+        // can verify it matches the approved version. Never echo the registry
+        // version; the trusted connector reports what it actually received.
+        return {
+          status: 200,
+          url: result.finalUrl,
+          version: result.version,
+          mediaType: result.contentType,
+          contentLength: result.bytes.length,
+          body: result.bytes,
+        };
+      },
+      maxBytes,
+    });
+  } catch (error) {
+    // An unavailable source is recorded as thrown, never as empty bytes.
+    // The caller may catch SourceIntakeError and record unavailability upstream.
+    throw error;
+  }
+
+  // 2. Check for reuse: does a snapshot with the same contentHash already exist?
+  const { snapshot, bytes } = captured;
+  const ns = parseNamespace(namespace);
+
+  // source-intake emits contentHash prefixed `sha256:` but the graph store
+  // computes its own raw hex hash from bytes (not caller-supplied). Drop the
+  // field so the graph store computes the hash, keeping the single source of truth.
+  const { contentHash: _ch, ...snapForStore } = snapshot;
+
+  let reused = false;
+  let persistedSnapshot = snapshot;
+  try {
+    const existing = await graphStore.listSourceSnapshots(caller, ns);
+    // graph store returns contentHash without prefix; source-intake snapshots have the prefix.
+    // Normalize for comparison: strip `sha256:` from the intake snapshot's hash.
+    const normalizedHash = snapshot.contentHash.startsWith("sha256:")
+      ? snapshot.contentHash.slice(7)
+      : snapshot.contentHash;
+    const match = existing.find((s) =>
+      s.contentHash === normalizedHash && s.sourceId === sourceId,
+    );
+    if (match) {
+      reused = true;
+      // Content bytes are identical — a snapshot with this (sourceId, version, contentHash)
+      // already exists. Return the existing record; do not write a duplicate.
+      // Normalize: graph store returns raw hex, but callers expect the sha256: prefix.
+      persistedSnapshot = { ...match, contentHash: "sha256:" + match.contentHash };
+    }
+  } catch {
+    // If listing fails, proceed with write
+  }
+
+  if (!reused) {
+    // 3. Write source snapshot to graph store (without caller-supplied contentHash;
+    //    the graph store computes its own from bytes).
+    await graphStore.writeSourceSnapshot(caller, ns, snapForStore, bytes);
+  }
+
+  // 4. Extract facts with byte offsets
+  let facts;
+  try {
+    facts = extractFromHtml(bytes);
+  } catch {
+    facts = { title: null, description: null, canonical: null, headings: [], links: [], jsonLd: [] };
+  }
+
+  return {
+    snapshot: persistedSnapshot,
+    facts,
+    reused,
+  };
+}