theprogrammablemind 9.7.1 → 9.8.0-beta.1

package/src/debug.js

@@ -27,6 +27,18 @@ const unhit = (name) => {
   hits[name] = false
 }
 
+const hitScoped = async (name, call) => {
+  hit(name)
+  try {
+    const result = await call()
+    unhit(name)
+    return result
+  } catch( e ) {
+    unhit(name)
+    throw e
+  }
+}
+
 const wasHit = (name) => {
   return hits[name]
 }
@@ -64,16 +76,19 @@ const _break = (name) => {
   }
 }
 
+const breakAt = _break;
+
 module.exports = {
   counter,
   get,
   _break, // break is a keyword
-
+  breakAt,
   // used for tests
   reset,
   hit,
   unhit,
   wasHit,
   getDb,
-  setDb
+  setDb,
+  hitScoped,
 }

package/src/digraph.js

@@ -73,11 +73,25 @@ class Digraph {
     this._edges = edges
   }
 
-  /*
-  set edges(edges) {
-    this._edges = edges.map( toA )
+  tos(from) {
+    const tos = []
+    for (const edge of this._edges) {
+      if (edge[0] == from) {
+        tos.push(edge[1])
+      }
+    }
+    return tos
+  }
+
+  froms(to) {
+    const froms = []
+    for (const edge of this._edges) {
+      if (edge[1] == to) {
+        froms.push(edge[0])
+      }
+    }
+    return froms
   }
-*/
 
   acdcs (s, from, to) {
     if (!s) {
@@ -170,18 +184,6 @@ class Digraph {
     return this.minima(common)
   }
 
-  /*
-  maxima (nodes) {
-    const maxima = new Set(nodes)
-    const descendants = new Set([])
-    nodes.forEach((node) => {
-      this.descendants(node).forEach((n) => descendants.add(n))
-    })
-    descendants.forEach((n) => maxima.delete(n))
-    return maxima
-  }
-  */
-
   add (child, parent) {
     this._edges.push([child, parent])
   }

package/src/digraph_internal.js

@@ -39,6 +39,26 @@ class DigraphInternal {
     this._edges = edges
   }
 
+  tos(from) {
+    const tos = []
+    for (const edge of this._edges) {
+      if (edge[0] == from) {
+        tos.push(edge[1])
+      }
+    }
+    return tos
+  }
+
+  froms(to) {
+    const froms = []
+    for (const edge of this._edges) {
+      if (edge[1] == to) {
+        froms.push(edge[0])
+      }
+    }
+    return froms
+  }
+
   get length () {
     return this._edges.length
   }
@@ -137,18 +157,6 @@ class DigraphInternal {
     return this.minima(common)
   }
 
-  /*
-  maxima (nodes) {
-    const maxima = new Set(nodes)
-    const descendants = new Set([])
-    nodes.forEach((node) => {
-      this.descendants(node).forEach((n) => descendants.add(n))
-    })
-    descendants.forEach((n) => maxima.delete(n))
-    return maxima
-  }
-  */
-
   add (child, parent) {
     this._edges.push([child, parent])
   }

package/src/flatten.js

@@ -1,7 +1,8 @@
-const { isArray, isObject, isCompound } = require('./helpers')
+const { isArray, isObject, isCompound, } = require('./helpers')
+const debug = require('./debug')
 const _ = require('lodash')
 
-const DEBUG = (...args) => {
+const DEBUG = (label, details) => {
 //  DEBUG(...args)
 }
 
@@ -46,33 +47,24 @@ const zip = (a1, a2) => {
 }
 
 const flattenListHelper = (markers, values) => {
-  const flats = []
   let wasFlattened = false
-  let singles = []
+  const all = []
   for (const value of values) {
     if (!isCompound(value)) {
-      singles.push(value)
+      all.push([value])
     } else {
       const [more, wf] = flatten(markers, value)
       if (wf) {
         wasFlattened = wasFlattened || wf
-        flats.push(more)
+        all.push(more)
       } else {
-        singles = singles.concat(more)
+        all.push(...more.map((m) => [m]))
       }
     }
   }
 
-  // product does weirdness with only choice
-  if (!wasFlattened) {
-    return [[singles], wasFlattened]
-  } else {
-    const results = []
-    for (const flat of cartesian(...flats)) {
-      results.push(singles.concat(flat))
-    }
-    return [results, wasFlattened]
-  }
+  const result = cartesian(...all)
+  return [result, wasFlattened]
 }
 
 // returns (list, wasFlattened)
@@ -84,6 +76,9 @@ const flatten = (markers, value) => {
   if (value.flatten === false) {
     return [[value], false]
   }
+
+  const flatten_ignore = value.flatten_ignore || []
+
   const marker = value.marker
   let properties = value
 
@@ -113,8 +108,9 @@ const flatten = (markers, value) => {
   for (const key in properties) {
     let wf = false
     let values
-    // if isinstance(properties[key], Context) {
-    if (isObject(properties[key])) {
+    if (flatten_ignore.includes(key)) {
+      values = [properties[key]]
+    } else if (isObject(properties[key])) {
       const context = properties[key];
       [values, wf] = flatten(markers, context)
     // } else if (isinstance(properties[key], list)) {

package/src/fragments.js

@@ -1,6 +1,28 @@
 const _ = require('lodash')
 const helpers = require('./helpers')
 
+function createChangeTracker(obj) {
+  const changes = new Set();
+
+  const handler = {
+    set(target, prop, value) {
+      changes.add(prop);
+      target[prop] = value;
+      return true;
+    },
+    deleteProperty(target, prop) {
+      changes.add(prop);
+      delete target[prop];
+      return true;
+    }
+  };
+
+  return {
+    proxy: new Proxy(obj, handler),
+    getChanges: () => Array.from(changes)
+  };
+}
+
 function fragmentInstantiator (args, contexts) {
   return new Object({ 
     contexts: () => {
@@ -13,13 +35,15 @@ function fragmentInstantiator (args, contexts) {
       args.pathEquals = helpers.pathEquals
       while (todo.length > 0) {
         const { context, path } = todo.pop()
-        args.context = context
+        const { proxy, getChanges } = createChangeTracker(context);
+        args.context = proxy
         args.path = path
         for (const mapping of mappings) {
           if (await mapping.match(args)) {
             await mapping.apply(args)
           }
         }
+        const ignore = getChanges()
         for (const key of Object.keys(context)) {
           // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
           if (!helpers.isCompound(context[key])) {
@@ -28,7 +52,9 @@ function fragmentInstantiator (args, contexts) {
           if (context[key].instantiated) {
             continue
           }
-          todo.push({ context: context[key], path: [...path, key] })
+          if (!ignore.includes(key)) {
+            todo.push({ context: context[key], path: [...path, key] })
+          }
         }
       }
       if (contexts.length == 1 && instantiated.length == 1) {

package/src/generators.js

@@ -1,6 +1,7 @@
 const { args: contextArgs, normalizeGenerator } = require('./helpers')
 const Lines = require('../lines')
 const helpers = require('./helpers')
+const debug = require('./debug')
 
 class Generator {
   // constructor ({ match, apply, uuid, index, km, priority, notes }) {
@@ -65,14 +66,16 @@ class Generator {
     }
     const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
     // return this.match(args)
-    const matches = await this.match(args)
-    if ((matches && (options.debug || {}).match) ||
-        callId === this.callId) {
-      // next line is the matcher
-      debugger // eslint-disable-line no-debugger
-      await this.match(args)
-    }
-    return matches
+    return await debug.hitScoped(args.callId, async () => {
+      const matches = await this.match(args)
+      if ((matches && (options.debug || {}).match) ||
+          callId === this.callId) {
+        // next line is the matcher
+        debugger // eslint-disable-line no-debugger
+        await this.match(args)
+      }
+      return matches
+    })
   }
 
   // apply (baseArgs, objects, g, gs, context, hierarchy, config, response, log, options = {}) {
@@ -121,11 +124,12 @@ class Generator {
     if (this.property === 'generatorp') {
       args.g = args.gp
     }
-    if ((options.debug || {}).apply ||
-        callId === this.callId) {
-      debugger // eslint-disable-line no-debugger
-    }
-    return await this._apply(args)
+    return await debug.hitScoped(args.callId, async () => {
+      if ((options.debug || {}).apply || process.env.DEBUG_CALLID == args.callId || callId === this.callId) {
+        debugger // eslint-disable-line no-debugger
+      }
+      return await this._apply(args)  // DEBUG: This is the line you want to step into
+    })
   }
 }
 
@@ -180,12 +184,22 @@ class Generators {
     let generated = ''
     let applied = false
     const stack = args.calls.push()
+    const pOrRTag = assumed.isResponse ? "(RESPONSE)" : "(PARAPHRASE)"
+    const finallyList = []; // called after the generator is processed
+    args._finally = (call) => finallyList.unshift(call)
     for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
       const generator = this.generators[igenerator]
       if (await generator.matches(args, objects, context, hierarchy, config, options)) {
         const log = (message) => { this.logs.push(message) }
         // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${stringify(context)}`)
         let errorMessage = 'The apply function did not return a value'
+        if (process.env.TPMKMS_TRACE) {
+          console.error("TPMKMS_TRACE", debug.counter("TPMKMS_TRACE"))
+          console.error(generator.toLabel())
+          console.error(generator.toString())
+          console.log(`To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_CALLID="${args.calls.current()}"`)
+        }
+
         try {
           generated = await generator.apply(args, objects, context, hierarchy, config, response, log, options)
         } catch (e) {
@@ -207,6 +221,7 @@ class Generators {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           const source = `${generator.km}/#${generator.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
           lines.newRow()
@@ -220,7 +235,7 @@ class Generators {
           lines.setElement(0, 2, stack)
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use args.callId === '${args.calls.current()}'`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_CALLID="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
@@ -235,6 +250,7 @@ class Generators {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           if (generator.index > -1 && generator.km) {
             // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
             lines.setElement(0, 2, generator.toLabel())
@@ -246,13 +262,13 @@ class Generators {
           lines.setElement(0, 2, generator.toString())
           lines.newRow()
           lines.setElement(0, 1, 'RESULT')
-          lines.setElement(0, 2, generated)
+          lines.setElement(0, 2, generated + " " + pOrRTag)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use args.callId === '${args.calls.current()}'`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
@@ -263,11 +279,15 @@ class Generators {
         break
       }
     }
+    for (const f of finallyList) {
+      await f()
+    }
     args.calls.pop()
     if (!applied && config.getDebug()) {
       const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Generator')
+      lines.setElement(0, 1, `#${context.context_id || "?"}`)
       lines.setElement(0, 2, 'No generator applied')
       lines.newRow()
       lines.setElement(0, 1, 'STACK')

package/src/helpers.js

@@ -1,6 +1,27 @@
 const deepEqual = require('deep-equal')
 const stringify = require('json-stable-stringify')
 
+
+// check if member function of a class have been overriden by an subclass
+class OverrideCheck {
+  constructor(base, suffix='API') {
+    this.base = base
+    this.checks = Object.getOwnPropertyNames(base.prototype).filter(key => typeof base.prototype[key] === 'function' && key.endsWith(suffix));
+  }
+
+  check(obj) {
+    const overriden = obj.constructor !== obj.baseAPI
+    if (!overriden) {
+      return
+    }
+    for (const check of this.checks) {
+      if (obj[check] == this.base.prototype[check]) {
+        throw new Error(`For ${obj.constructor.name} you need to override ${check}`)
+      }
+    }
+  }
+}
+
 function pathEquals (p1, p2) {
   if (p1.length !== p2.length) {
     return false
@@ -523,52 +544,77 @@ const stableId = (tag) => {
 }
 
 function getByPath(obj, path, defaultValue) {
+  const segments = Array.isArray(path) ? path : [path];
+  const keys = segments.flatMap((segment) => {
+    if (typeof segment !== "string") return [segment];
+    return segment
+      .replace(/\[(\d+)\]/g, ".$1")
+      .split(".")
+      .filter((key) => key !== "");
+  });
+
   let current = obj;
-  for (const key of path) {
+  for (const key of keys) {
     if (current === null || current === undefined) return defaultValue;
-    if (typeof current !== 'object') return defaultValue;
+    if (typeof current !== "object") return defaultValue;
     current = current[key];
   }
   return current === undefined ? defaultValue : current;
 }
 
+function normalizePath(path) {
+  const segments = Array.isArray(path) ? path : [path];
+  return segments.flatMap((segment) => {
+    if (typeof segment !== "string") return [segment];
+    return segment
+      .replace(/\[(\d+)\]/g, ".$1")
+      .split(".")
+      .filter((key) => key !== "");
+  });
+}
+
+function isArrayIndex(key) {
+  if (typeof key === "number") return Number.isInteger(key) && key >= 0;
+  return key !== "" && String(key >>> 0) === String(key);
+}
+
 /**
- * Set a value in an object by path array.
+ * Set a value in an object by path (string or array).
  * Automatically creates missing objects {} or arrays [] as needed.
  *
  * @param {Object} obj - The root object to modify
- * @param {Array<string|number>} path - Array of keys/indices
+ * @param {string|Array<string|number>} path - Path keys/indices
  * @param {*} value - Value to set
  * @returns {*} The set value (for chaining)
  */
 function setByPath(obj, path, value) {
-  if (!Array.isArray(path) || path.length === 0) {
-    throw new Error('Path must be a non-empty array');
+  const keys = normalizePath(path);
+
+  if (keys.length === 0) {
+    throw new Error("Path must be a non-empty string or array");
+  }
+
+  if (obj == null || typeof obj !== "object") {
+    throw new Error("Target must be an object or array");
   }
 
   let current = obj;
 
-  for (let i = 0; i < path.length; i++) {
-    const key = path[i];
-    const isLast = i === path.length - 1;
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    const isLast = i === keys.length - 1;
 
     if (isLast) {
-      // Final step — just assign
       current[key] = value;
     } else {
-      // Not last — ensure next level exists
-      const nextKey = path[i + 1];
-
-      if (current[key] == null) {
-        // Auto-create: array if next key is number, otherwise object
-        current[key] = typeof nextKey === 'number' || String(nextKey >>> 0) === nextKey
-          ? []
-          : {};
-      } else if (Array.isArray(current[key]) && typeof nextKey !== 'number') {
-        // Safety: if current is array but next key isn't a valid index → convert to object
+      const nextKey = keys[i + 1];
+      const nextIsIndex = isArrayIndex(nextKey);
+
+      if (current[key] == null || typeof current[key] !== "object") {
+        current[key] = nextIsIndex ? [] : {};
+      } else if (Array.isArray(current[key]) && !nextIsIndex) {
         current[key] = { ...current[key] };
-      } else if (!Array.isArray(current[key]) && typeof nextKey === 'number') {
-        // If next expects array but current is object → convert
+      } else if (!Array.isArray(current[key]) && nextIsIndex) {
         current[key] = Object.values(current[key]);
       }
 
@@ -579,7 +625,64 @@ function setByPath(obj, path, value) {
   return value;
 }
 
+const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'["cat", "mammel"]\''
+
+// removeProp.js
+function removeProp(obj, testFn, { maxDepth = Infinity, seen = new WeakSet() } = {}) {
+  if (!obj || typeof obj !== 'object' || maxDepth <= 0) return obj;
+  if (seen.has(obj)) return obj;
+  seen.add(obj);
+
+  if (Array.isArray(obj)) {
+    // ---- ARRAY: process each element (but don't remove elements unless testFn says so)
+    let writeIdx = 0;
+    for (let i = 0; i < obj.length; i++) {
+      const element = obj[i];
+      const shouldRemoveElement = testFn(element, i, obj);
+
+      if (shouldRemoveElement) {
+        // Remove the whole array element
+        if (element && typeof element === 'object') {
+          // Still walk inside it in case testFn wants side effects
+          removeProp(element, testFn, { maxDepth: maxDepth - 1, seen });
+        }
+        // Skip writing it back
+      } else {
+        // Keep element, but walk into it to remove inner props
+        removeProp(element, testFn, { maxDepth: maxDepth - 1, seen });
+        if (writeIdx !== i) {
+          obj[writeIdx] = element;
+        }
+        writeIdx++;
+      }
+    }
+    obj.length = writeIdx;
+    return obj;
+  }
+
+  // ---- OBJECT: iterate over own keys
+  const keys = Reflect.ownKeys(obj);
+  for (const key of keys) {
+    const val = obj[key];
+    const shouldRemove = testFn(val, key, obj);
+
+    if (shouldRemove) {
+      delete obj[key];
+      if (val && typeof val === 'object') {
+        removeProp(val, testFn, { maxDepth: maxDepth - 1, seen });
+      }
+    } else {
+      removeProp(val, testFn, { maxDepth: maxDepth - 1, seen });
+    }
+  }
+
+  return obj;
+}
+
 module.exports = {
+  removeProp,
+  helpDebugHierarchy,
+
   stableId,
   ecatch,
   functionsToStrings,
@@ -611,4 +714,5 @@ module.exports = {
   assignAssumed,
   watchProperty,
   pathEquals,
+  OverrideCheck,
 }

package/src/project2.js

@@ -101,6 +101,10 @@ function project(source, filters, path=[]) {
     } else if (prop.property && source.hasOwnProperty(prop.property)) {
       // If the property is an object and not null, recursively project it
       if (typeof source[prop.property] === 'object' && source[prop.property] !== null) {
+        if (prop.all) {
+          result[prop.property] = source[prop.property]
+          return
+        }
         result[prop.property] = {}
         const instantiatedCheck = []
         for (const check of prop.check) {