statebus 6.1.12 → 6.1.16

package/client.js

@@ -60,6 +60,7 @@
     function braid_http_mount (prefix, url) {
         var preprefix = prefix.slice(0,-1)
         var has_prefix = new RegExp('^' + preprefix)
+        var is_absolute = /^https?:\/\//
         var client_fetched_keys = new bus.Set()
         if (url[url.length-1]=='/') url = url.substr(0,url.length-1)
         function add_prefix (key) {

package/extras/tests.js

@@ -294,6 +294,43 @@ test(function url_translation (done) {
     done()
 })
 
+test(function translate_fields (done) {
+    // Translate Statebus -> Proxy format
+    var tests1 = [
+        [{key: 'foo', link: 'boo'}, {key: 'foo', link: 'boo'}],
+        [{_key: 'foo', link: 'boo'}, {key: 'foo', link: 'boo'}],
+        [{__key: 'foo', link: 'boo'}, {_key: 'foo', link: 'boo'}],
+        [[{_key: 'foo', link: 'boo'}], [{key: 'foo', link: 'boo'}]],
+        [{a: [{_key: 'foo'}]}, {a: [{key: 'foo'}]}]
+    ]
+
+    // Translate Proxy -> Statebus format
+    var tests2 = [
+        [{key: 'foo', link: 'boo'}, {_key: 'foo', link: 'boo'}],
+        [{_key: 'foo', link: 'boo'}, {__key: 'foo', link: 'boo'}],
+        [[{_key: 'foo', link: ['boo']}], [{__key: 'foo', link: ['boo']}]]
+    ]
+
+    // Test 1
+    var tests = tests1
+    for (var i=0; i<tests.length; i++) {
+        var trans = bus.translate_fields(tests[i][0], bus.keyed_2_proxied)
+        assert(bus.deep_equals(trans, tests[i][1]),
+               'Bad translation1: ' + i + ' ' + JSON.stringify(trans))
+    }
+
+    // Test 2
+    tests = tests2
+    for (var i=0; i<tests2.length; i++) {
+        var trans = bus.translate_fields(tests[i][0], bus.proxied_2_keyed)
+        assert(bus.deep_equals(trans, tests[i][1]),
+               'Bad translation2: ' + i + ' ' + JSON.stringify(trans))
+    }
+
+    log('Passed translate_fields')
+    done()
+})
+
 test(function basics (done) {
     bus('basic wait').to_fetch = function () {
         setTimeout(function () {bus.save.fire({key:'basic wait', a:1})},
@@ -834,6 +871,102 @@ test(function proxies2 (done) {
     // Getting a normal property should do a fetch
 })
 
+test(function braid_proxies (done) {
+    var bus = require('../statebus').serve({braid_mode_test: true,
+                                            file_store: false})
+    var state = bus.state
+
+    assert(state.array === undefined)
+    assert(state.bar === undefined)
+
+    state.array = []
+    log('array:', state.array)
+    assert(state.array.length === 0)
+    state.array[0] = 1
+    log('array:', state.array)
+    assert(state.array.length === 1)
+    state.bar = {}
+    log('bar:', state.bar)
+    state.bar = {a: 1}
+    log('bar:', state.bar)
+    assert(state.bar.a === 1)
+    state.bar.a = state.array
+    log('bar:', state.bar)
+    state.array[1] = 2
+    log('array:', state.array)
+    log('bar:', state.bar)
+    assert(state.bar.a[1] !== 2,
+           "Array ref linked.\n\t"
+           + JSON.stringify(bus.cache.bar) + '\n\t'
+           + JSON.stringify(bus.cache.array))
+
+    state.undefining = undefined
+    assert(state.undefining === undefined)
+    state.undefining = {a: undefined}
+    log('state.undefining =', state.undefining)
+    // assert(state.undefining.a === undefined)
+    // TODO: fix https://github.com/invisible-college/statebus/issues/34
+    state.undefining = {}
+    assert(!('a' in state.undefining))
+    state.undefining.a = undefined
+    assert('a' in state.undefining)
+    assert(state.undefining.a === undefined)
+
+    state.b = {a: undefined}
+    assert('a' in state.b)
+    assert(state.b.a === undefined)
+    delete state.b.a
+    // TODO: https://github.com/invisible-college/statebus/issues/34
+    assert(!('a' in state.b))
+    assert(state.b.a === undefined)
+
+    return done()
+
+    /*
+      Things I want to test:
+
+      - Setting nested items
+      - Escaping their fields
+      - Calling fetch on them
+      - Converting state[..] to keyed objects internally
+      - has() potentially doing a fetch, or loading()
+      - set() returning a proxy object
+      - console output
+      - node AND chrome
+      - having nice colors and distinctions and shit
+    */
+
+    state.foo = 3
+    // This should set into _
+    console.assert(bus.validate(bus.cache.foo,
+                                {key: 'foo', _: 3}))
+
+
+    state.foo = {a: 5}
+    // This should set directly on it
+    console.assert(bus.validate(bus.cache.foo,
+                                {key: 'foo', a: 5}))
+    state.foo.b = 6
+    console.assert(bus.validate(bus.cache.foo,
+                                {key: 'foo', b: 6}))
+
+    state.bar = [3]
+    state.foo = {a: 3, bar: state.bar}
+
+    bus(() => {
+        state.foo      // foo triggers re-render
+        state.foo.bar  // bar triggers re-render
+        state.foo.a    // triggers re-render too
+    })
+
+    // Getting a linked item should do a fetch
+    bus(() => {
+        state.bar
+    })
+
+    // Getting a normal property should do a fetch
+})
+
 test(function only_one (done) {
     bus('only_one/*').to_fetch = function (k) {
         var id = k[k.length-1]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "statebus",
-  "version": "6.1.12",
+  "version": "6.1.16",
   "description": "Synchronize state.",
   "main": "statebus.js",
   "scripts": {

package/server.js

@@ -2,6 +2,15 @@ var fs = require('fs'),
     util = require('util')
 var unique_sockjs_string = '_connect_to_statebus_'
 
+// Import braidify if it's available
+try {
+    var braidify = require('braidify').http_server
+    console.log('Found braidify library. Braid-HTTP enabled!')
+} catch (e) {
+    var braidify = undefined
+}
+
+
 function default_options (bus) { return {
     port: 'auto',
     backdoor: null,
@@ -68,6 +77,7 @@ function import_server (bus, options)
         bus.express = express()
         bus.http = express.Router()
         bus.install_express(bus.express)
+        bus.initialize_braid_http()
 
         // use gzip compression if available
         try { bus.http.use(require('compression')())
@@ -99,16 +109,94 @@ function import_server (bus, options)
         // User will put his routes in here
         bus.express.use('/', bus.http)
 
-        // Use braidify if it's available
-        try {
-            var braidify = require('braidify').http_server
-            console.log('Found braidify library. Braid-HTTP enabled!')
-        } catch (e) {
-            var braidify = undefined
+        // Free the CORS for Braid requests!
+        bus.express.use((req, res, next) => {
+            // If this requests knows about Braid then we presume the
+            // programmer knows that any client might make this cross-origin
+            // request.
+            if (req.version || req.parents || req.subscribe
+                || req.headers.subscribe
+                || req.method === 'OPTIONS')
+                free_the_cors(req, res, next)
+            else
+                next()
+        })
+
+        // Connect bus to the HTTP server
+        bus.express.use(bus.http_handlers)
+
+        // Serve Client Coffee
+        bus.serve_client_coffee()
+
+        // Custom router
+        bus.route = function server_route (key, method, arg, t) {
+            var handlers = bus.bindings(key, method)
+            var count = handlers.length
+            if (count)
+                bus.log('route:', bus+'("'+key+'").'+method+'['+handlers.length+'](key:"'+(arg.key||arg)+'")')
+
+            // Call all handlers!
+            for (var i=0; i<handlers.length; i++)
+                bus.run_handler(handlers[i].func, method, arg, {t: t, binding: handlers[i].key})
+
+            // Now handle backup handlers
+            if (count === 0) {
+
+                // A save to master without a to_save defined should be
+                // automatically reflected to all clients.
+                if (method === 'to_save') {
+                    bus.save.fire(arg, t)
+                    bus.route(arg.key, 'on_set_sync', arg, t)
+                    count++
+                }
+
+                // A fetch to master without a to_fetch defined should go to
+                // the database, if we have one.
+                if (method === 'to_fetch') {
+                    bus.db_fetch(key)
+                    // This basically renders the count useless-- you probably
+                    // don't want to wrap this route() with another route().
+                    count++
+                }
+            }
+            return handlers.length
         }
 
-        // Mirror state over HTTP
-        bus.express.get('*', function (req, res) {
+        // Back door to the control room
+        if (bus.options.backdoor) {
+            bus.make_http_server({
+                port: bus.options.backdoor,
+                name: 'backdoor_http_server',
+                use_ssl: use_ssl
+            })
+            bus.sockjs_server(this.backdoor_http_server)
+        }
+    },
+
+    initialize_braid_http: function () {
+        // Set up default linked json converters
+        if (bus.options.braid_mode_test) {
+            bus.to_http_body = (o) => JSON.stringify(o.val)
+            bus.state = bus.braid_proxy()
+        } else
+            bus.to_http_body = (o) => {
+                var unwrap = (Object.keys(o).length === 2
+                              && '_' in o
+                              && typeof o._ === 'string')
+                // To do: translate pointers as keys
+                return unwrap ? o._ : JSON.stringify(o)
+            }
+
+        bus.from_http_body = (key, body) => ({
+            key,
+            val: JSON.parse(body)
+        })
+    },
+
+    // Connect HTTP GET and PUT to our Fetch and Save
+    http_handlers: function (req, res, next) {
+        if (req.method === 'GET') {
+
             // Make a temporary client bus
             var cbus = bus.bus_for_http_client(req, res)
             var cb
@@ -146,13 +234,13 @@ function import_server (bus, options)
                 else
                     res.send(body)
 
-                // And shut down the connection if there's nothing left to do
+                // And shut down the connection if nothing's left to do
                 if (!braidify || !req.subscribe || !body)
                     end_it_all()
             })
-        })
+        }
 
-        bus.express.put('*', function (req, res) {
+        else if (req.method === 'PUT') {
             console.log('Got a put with body', req.body)
 
             // Make a temporary client bus
@@ -190,72 +278,9 @@ function import_server (bus, options)
                     res.end()
                 })
             }
-        })
-
-        // Set up default linked json converters
-        if (bus.options.braid_mode_test) {
-            bus.to_http_body = (o) => JSON.stringify(o.val)
-            bus.state = bus.braid_proxy()
-        } else
-            bus.to_http_body = (o) => {
-                var unwrap = (Object.keys(o).length === 2
-                              && '_' in o
-                              && typeof o._ === 'string')
-                // To do: translate pointers as keys
-                return unwrap ? o._ : JSON.stringify(o)
-            }
-
-        bus.from_http_body = (key, body) => ({
-            key,
-            val: JSON.parse(body)
-        })
-
-        // Serve Client Coffee
-        bus.serve_client_coffee()
-
-        // Custom router
-        bus.route = function server_route (key, method, arg, t) {
-            var handlers = bus.bindings(key, method)
-            var count = handlers.length
-            if (count)
-                bus.log('route:', bus+'("'+key+'").'+method+'['+handlers.length+'](key:"'+(arg.key||arg)+'")')
-
-            // Call all handlers!
-            for (var i=0; i<handlers.length; i++)
-                bus.run_handler(handlers[i].func, method, arg, {t: t, binding: handlers[i].key})
-
-            // Now handle backup handlers
-            if (count === 0) {
-
-                // A save to master without a to_save defined should be
-                // automatically reflected to all clients.
-                if (method === 'to_save') {
-                    bus.save.fire(arg, t)
-                    bus.route(arg.key, 'on_set_sync', arg, t)
-                    count++
-                }
-
-                // A fetch to master without a to_fetch defined should go to
-                // the database, if we have one.
-                if (method === 'to_fetch') {
-                    bus.db_fetch(key)
-                    // This basically renders the count useless-- you probably
-                    // don't want to wrap this route() with another route().
-                    count++
-                }
-            }
-            return handlers.length
-        }
-
-        // Back door to the control room
-        if (bus.options.backdoor) {
-            bus.make_http_server({
-                port: bus.options.backdoor,
-                name: 'backdoor_http_server',
-                use_ssl: use_ssl
-            })
-            bus.sockjs_server(this.backdoor_http_server)
         }
+        else
+            next()
     },
 
     bus_for_http_client: function (req, res) {
@@ -2450,6 +2475,24 @@ function import_server (bus, options)
 }
 
 
+// Disables CORS in HTTP servers
+function free_the_cors (req, res, next) {
+    console.log('free the cors!', req.method, req.url)
+
+    var free_the_cors = {
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Methods": "OPTIONS, HEAD, GET, PUT, UNSUBSCRIBE",
+        "Access-Control-Allow-Headers": "subscribe, peer, version, parents, merge-type, content-type, patches, cache-control"
+    }
+    Object.entries(free_the_cors).forEach(x => res.setHeader(x[0], x[1]))
+    if (req.method === 'OPTIONS') {
+        res.writeHead(200)
+        res.end()
+    } else
+        next()
+}
+
+
 // Handy functions for writing tests on nodejs
 var tests = []
 function test (f) {tests.push(f)}

package/statebus.js

@@ -1217,19 +1217,19 @@
                     get: function get(o, k) {
                         if (k === 'inspect' || k === 'valueOf' || typeof k === 'symbol')
                             return undefined
-                        k = underscore_escape(k)
+                        k = escape_keys(k)
                         return item_proxy(base, o[k])
                     },
                     set: function set(o, k, v) {
-                        var result = o[underscore_escape(k)] = v
+                        var result = o[escape_keys(k)] = v
                         bus.save(base)
                         return result
                     },
                     has: function has(o, k) {
-                        return o.hasOwnProperty(underscore_escape(k))
+                        return o.hasOwnProperty(escape_keys(k))
                     },
                     deleteProperty: function del (o, k) {
-                        delete o[underscore_escape(k)]
+                        delete o[escape_keys(k)]
                     },
                     apply: function apply (o, This, args) {
                         return o
@@ -1268,7 +1268,7 @@
                 // },
                 // ... but I haven't had a need yet.
                 deleteProperty: function del (o, k) {
-                    bus.delete(underscore_escape(k))
+                    bus.delete(escape_keys(k))
                 }
             })
         })()
@@ -1362,7 +1362,7 @@
                 // Actual objects need their keys translated
                 var result = {}
                 for (var k in x)
-                    result[underscore_escape(k)] = proxy_encode_val(x[k])
+                    result[escape_keys(k)] = proxy_encode_val(x[k])
                 return result
             }
 
@@ -1396,7 +1396,7 @@
                 var obj = {}
                 for (var k in json)
                     if (k !== 'key')
-                        obj[underscore_unescape(k)] = proxy_decode_json(json[k])
+                        obj[unescape_keys(k)] = proxy_decode_json(json[k])
                 return obj
             }
 
@@ -1452,7 +1452,7 @@
                         return undefined
                     }
 
-                    var tmp2 = pget(base, o, underscore_escape(k))
+                    var tmp2 = pget(base, o, escape_keys(k))
                     var base2 = tmp2[0]
                     var o2 = tmp2[1]
 
@@ -1463,7 +1463,7 @@
                     // console.log('set:', {base, o, k, v})
 
                     if (base) {
-                        var encoded_v = o[underscore_escape(k)] = proxy_encode_val(v)
+                        var encoded_v = o[escape_keys(k)] = proxy_encode_val(v)
                         // console.log('  set: saving', encoded_v, 'into', base)
 
                         // Collapse state of the form:
@@ -1523,18 +1523,18 @@
                     //  - Does this need to do a fetch as well?
                     //
                     //  - For a keyed object, should this do a loading() check?
-                    return o.hasOwnProperty(underscore_escape(k))
+                    return o.hasOwnProperty(escape_keys(k))
                 },
                 deleteProperty: function del (O, k) {
                     if (base) {
-                        // console.log('  deleting:', underscore_escape(k), 'of', o)
-                        delete o[underscore_escape(k)]   // Deleting innards
+                        // console.log('  deleting:', escape_keys(k), 'of', o)
+                        delete o[escape_keys(k)]   // Deleting innards
                         if (Object.keys(o).length === 1 && o.key)
                             o._ = {}
                         bus.save(base)
                     }
                     else
-                        bus.delete(underscore_escape(k)) // Deleting top-level
+                        bus.delete(escape_keys(k)) // Deleting top-level
                 },
                 apply: function apply (f, This, args) { return get_json() }
             })
@@ -1546,9 +1546,12 @@
         if (!nodejs)
             window.devtoolsFormatters = [{
                 header: function (x) {
-                    return x[symbols.is_proxy] &&
-                        ['span', {style: 'background-color: #feb; padding: 3px;'},
-                         JSON.stringify(proxy_decode_json(x()))]
+                    if (x[symbols.is_proxy])
+                        return ['span', {style: 'background-color: #feb; padding: 3px;'},
+                                JSON.stringify(proxy_decode_json(x()))]
+                    else if (x[symbols.is_braid_proxy])
+                        return ['span', {style: 'background-color: #fffbe5; padding: 3px;'},
+                                JSON.stringify(x)]
                 },
                 hasBody: function (x) {return false}
             }]
@@ -1556,41 +1559,61 @@
 
     // ******************
     // Braid Test Mode Proxy
-    function braid_proxy () {
-        // I have the cache behind the scenes
-        // Each proxy has a target object -- the raw data on cache
-        // If we're proxying a {_: ...} singleton then ...
 
+    var symbols = {is_braid_proxy: Symbol('is_braid_proxy'),
+                   is_proxy: Symbol('is_proxy'),
+                   is_link: Symbol('is_link'),
+                   get_json: Symbol('get_json'),
+                   get_base: Symbol('get_base')
+                  }
+    function braid_proxy () {
         function item_proxy (base, o) {
+
+            // We only create a proxy for objects.
             if (typeof o === 'number'
                 || typeof o === 'string'
                 || typeof o === 'boolean'
                 || o === undefined
                 || o === null
-                || typeof o === 'function') return o
+                || typeof o === 'function')
+                // Everything else passes through unscathed
+                return o
+
+            // We recursively descend through {key: ...} links
+            if (typeof o === 'object' && 'key' in o) {
+                var new_base = bus.fetch(o.key)
+                return item_proxy(new_base, new_base.val)
+            }
 
             return new Proxy(o, {
                 get: function get(o, k) {
-                    if (k === 'inspect' || k === 'valueOf' || typeof k === 'symbol')
+                    if (k === 'inspect' || k === 'valueOf')
                         return undefined
-                    return item_proxy(base, o[underscore_escape(k)])
+                    if (k === symbols.is_braid_proxy)
+                        return true
+                    if (typeof k === 'symbol')
+                        return undefined
+                    return item_proxy(base, o[proxied_2_keyed(k)])
                 },
                 set: function set(o, k, v) {
-                    var result = o[underscore_escape(k)] = v
+                    var value = translate_fields(v, proxied_2_keyed)
+                    o[proxied_2_keyed(k)] = value
                     bus.save(base)
-                    return result
+                    return true
                 },
                 has: function has(o, k) {
-                    return o.hasOwnProperty(underscore_escape(k))
+                    return o.hasOwnProperty(proxied_2_keyed(k))
                 },
                 deleteProperty: function del (o, k) {
-                    delete o[underscore_escape(k)]
+                    delete o[proxied_2_keyed(k)]
                 },
                 apply: function apply (o, This, args) {
                     return o
                 }
             })}
 
+
+        // The top-level Proxy object holds HTTP resources
         return new Proxy(cache, {
             get: function get(o, k) {
                 if (k === 'inspect' || k === 'valueOf' || typeof k === 'symbol')
@@ -1601,21 +1624,27 @@
                 return item_proxy(base, base.val)
             },
             set: function set(o, key, val) {
-                bus.save({key, val})
+                bus.save({key: key,
+                          val: translate_fields(val, proxied_2_keyed)})
+                return true
             },
-            // In future, this might check if there's a .to_fetch function OR
-            // something in the cache:
-            //
-            // has: function has(o, k) {
-            //     return k in o
-            // },
-            // ... but I haven't had a need yet.
             deleteProperty: function del (o, k) {
-                bus.delete(underscore_escape(k))
+                bus.delete(proxied_2_keyed(k))
             }
         })
     }
 
+    function link (url) {
+        var result = fetch(url)
+        result[symbols.is_link] = true
+        return result
+    }
+
+    if ((nodejs && bus.options && bus.options.braid_mode_test)
+        || (!nodejs && window.Proxy && clientjs_option('braid_mode_test')))
+        state = braid_proxy()
+
+
     // ******************
     // Network client
     function get_domain (key) { // Returns e.g. "state://foo.com"
@@ -1822,8 +1851,8 @@
     }
 
 
-    // ******************
-    // Key translation
+    // ************************************************
+    // Translating the URLs under keys of state
     function translate_keys (obj, f) {
         // Recurse through each element in arrays
         if (Array.isArray(obj))
@@ -1845,14 +1874,60 @@
             }
         return obj
     }
-    function underscore_escape(k) {
+    function escape_keys(k) {
         return k.replace(/(_(keys?|time)?$|^key$)/, '$1_')
     }
-    function underscore_unescape (k) {
+    function unescape_keys (k) {
         return k.replace(/(_$)/, '')
     }
 
 
+    // ************************************************
+    // Translating fields of objects
+
+    // Recurse through JSON and swap all object fields with other field names,
+    // according to the function f(field, object).  Returns a copy.
+    //
+    // f(field, object) takes a string `field` and returns the new field name
+    // for `object`.
+    function translate_fields (input, f) {
+        var result
+
+        // Recurse through each element in arrays
+        if (Array.isArray(input)) {
+            var new_array = input.slice()
+            for (var i=0; i < input.length; i++)
+                new_array[i] = translate_fields(input[i], f)
+            result = new_array
+        }
+
+        // Recurse through each property on objects
+        else if (typeof input === 'object') {
+            var new_obj = {}
+            for (var k in input)
+                new_obj[f(k, input)] = translate_fields(input[k], f)
+            result = new_obj
+        }
+
+        // Everything else passes through unscathed
+        else
+            result = input
+
+        return result
+    }
+    // Remove underscore from _key
+    function keyed_2_proxied (k, object) {
+        if (object[symbols.is_link])
+            return k
+        else
+            return k.replace(/^(_*)_key$/, '$1key')
+    }
+    // Add underscore to key
+    function proxied_2_keyed (k) {
+        return k.replace(/^(_*)key$/, '$1_key')
+    }
+
+
     // ******************
     // JSON encoding
     //  - Does both escape/unescape keys, and wraps/unwraps pointers
@@ -2299,13 +2374,14 @@
     var api = ['cache backup_cache fetch save forget del fire dirty fetch_once',
                'subspace bindings run_handler bind unbind reactive uncallback',
                'versions new_version',
-               'make_proxy braid_proxy state sb',
+               'make_proxy braid_proxy state sb link',
                'funk_key funk_name funks key_id key_name id',
                'pending_fetches fetches_in fetches_out loading_keys loading once',
                'global_funk busses rerunnable_funks',
-               'underscore_escape underscore_unescape translate_keys apply_patch',
+               'escape_keys unescape_keys translate_keys apply_patch',
+               'keyed_2_proxied proxied_2_keyed translate_fields',
                'net_mount net_automount message_method',
-               'parse Set One_To_Many clone extend deep_map deep_equals prune validate sorta_diff log deps'
+               'parse Set One_To_Many clone extend deep_map deep_equals prune validate sorta_diff log deps symbols'
               ].join(' ').split(' ')
     for (var i=0; i<api.length; i++)
         bus[api[i]] = eval(api[i])