synapse 3.1.0__tar.gz → 3.3.0__tar.gz

{synapse-3.1.0/synapse.egg-info → synapse-3.3.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 3.1.0
+Version: 3.3.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://vertex.link
-Project-URL: Documentation, https://docs.vertex.link
+Project-URL: Documentation, https://hub.vertex.link/docs
 Project-URL: Repository, https://github.com/vertexproject/synapse
-Project-URL: Changelog, https://docs.vertex.link/docs/synapse/latest/changelog.html
+Project-URL: Changelog, https://hub.vertex.link/docs/synapse/latest/changelog.md
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Topic :: Database :: Database Engines/Servers
 Classifier: Topic :: System :: Clustering
@@ -68,7 +68,7 @@ Dynamic: license-file
 
 Synapse is a versatile central intelligence system created to support analyst teams in every stage of the intelligence life cycle.
 
-[![codecov](https://codecov.io/gh/vertexproject/synapse/branch/master/graph/badge.svg?branch=master)](https://v.vtx.lk/synapse-codecov) [![circleci](https://circleci.com/gh/vertexproject/synapse/tree/master.svg?style=svg)](https://v.vtx.lk/synapse-circleci) [![rtd](https://readthedocs.com/projects/vertex-synapse/badge/?version=latest)](https://v.vtx.lk/synapse-docs)
+[![codecov](https://codecov.io/gh/vertexproject/synapse/branch/master/graph/badge.svg?branch=master)](https://v.vtx.lk/synapse-codecov) [![circleci](https://circleci.com/gh/vertexproject/synapse/tree/master.svg?style=svg)](https://v.vtx.lk/synapse-circleci)
 
 ## Learn More About Synapse
 
@@ -77,10 +77,10 @@ Synapse is a versatile central intelligence system created to support analyst te
 
 ## Installation & Documentation
 
-- Synapse is available as a Python Package on [pypi](https://v.vtx.lk/synapse-pypi) and as a Docker image on [dockerhub](https://v.vtx.lk/synapse-dockerhub).
-  - The Getting Started [guide](https://docs.vertex.link/docs/synapse/latest/quickstart.html) will help get you up and running!
-- You can also use [synapse_quickstart](https://github.com/vertexproject/synapse-quickstart) to get up and running quickly on Windows, Mac, or Linux and start using Synapse!
-- User Guides and API Documentation can be found [here](https://v.vtx.lk/synapse-docs).
+- Synapse is available as a Python Package on [pypi](https://v.vtx.lk/synapse-pypi) and as a Docker image on [dockerhub](https://v.vtx.lk/synapse-dockerhub) and
+[Vertex Hub](https://hub.vertex.link/hub).
+  - The Getting Started [guide](https://hub.vertex.link/docs/synapse/latest/getting_started.md#syn-quick) will help get you up and running!
+- User Guides and API Documentation can be found on the [Vertex Hub](https://hub.vertex.link/docs).
 
 ## Connect With Us
 

{synapse-3.1.0 → synapse-3.3.0}/README.md

@@ -2,7 +2,7 @@
 
 Synapse is a versatile central intelligence system created to support analyst teams in every stage of the intelligence life cycle.
 
-[![codecov](https://codecov.io/gh/vertexproject/synapse/branch/master/graph/badge.svg?branch=master)](https://v.vtx.lk/synapse-codecov) [![circleci](https://circleci.com/gh/vertexproject/synapse/tree/master.svg?style=svg)](https://v.vtx.lk/synapse-circleci) [![rtd](https://readthedocs.com/projects/vertex-synapse/badge/?version=latest)](https://v.vtx.lk/synapse-docs)
+[![codecov](https://codecov.io/gh/vertexproject/synapse/branch/master/graph/badge.svg?branch=master)](https://v.vtx.lk/synapse-codecov) [![circleci](https://circleci.com/gh/vertexproject/synapse/tree/master.svg?style=svg)](https://v.vtx.lk/synapse-circleci)
 
 ## Learn More About Synapse
 
@@ -11,10 +11,10 @@ Synapse is a versatile central intelligence system created to support analyst te
 
 ## Installation & Documentation
 
-- Synapse is available as a Python Package on [pypi](https://v.vtx.lk/synapse-pypi) and as a Docker image on [dockerhub](https://v.vtx.lk/synapse-dockerhub).
-  - The Getting Started [guide](https://docs.vertex.link/docs/synapse/latest/quickstart.html) will help get you up and running!
-- You can also use [synapse_quickstart](https://github.com/vertexproject/synapse-quickstart) to get up and running quickly on Windows, Mac, or Linux and start using Synapse!
-- User Guides and API Documentation can be found [here](https://v.vtx.lk/synapse-docs).
+- Synapse is available as a Python Package on [pypi](https://v.vtx.lk/synapse-pypi) and as a Docker image on [dockerhub](https://v.vtx.lk/synapse-dockerhub) and
+[Vertex Hub](https://hub.vertex.link/hub).
+  - The Getting Started [guide](https://hub.vertex.link/docs/synapse/latest/getting_started.md#syn-quick) will help get you up and running!
+- User Guides and API Documentation can be found on the [Vertex Hub](https://hub.vertex.link/docs).
 
 ## Connect With Us
 

{synapse-3.1.0 → synapse-3.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '3.1.0'
+version = '3.3.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -76,9 +76,9 @@ build = [
 
 [project.urls]
 Homepage = 'https://vertex.link'
-Documentation = 'https://docs.vertex.link'
+Documentation = 'https://hub.vertex.link/docs'
 Repository = 'https://github.com/vertexproject/synapse'
-Changelog = 'https://docs.vertex.link/docs/synapse/latest/changelog.html'
+Changelog = 'https://hub.vertex.link/docs/synapse/latest/changelog.md'
 
 [tool.setuptools]
 include-package-data = true

{synapse-3.1.0 → synapse-3.3.0}/synapse/axon.py

@@ -616,7 +616,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
             proxy (str|bool): The proxy value.
 
         Notes:
-            The response body will be stored, regardless of the response code. The ``ok`` value in the response does not
+            The response body will be stored, regardless of the response code. The `ok` value in the response does not
             reflect that a status code, such as a 404, was encountered when retrieving the URL.
 
             The ssl dictionary may contain the following values::
@@ -1664,7 +1664,8 @@ class Axon(s_cell.Cell):
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
-        async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
+        sesshdrs = {'User-Agent': self.getUserAgent()}
+        async with aiohttp.ClientSession(connector=connector, timeout=atimeout, headers=sesshdrs) as sess:
             try:
                 data = aiohttp.FormData()
                 data._is_multipart = True
@@ -1737,7 +1738,8 @@ class Axon(s_cell.Cell):
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
-        async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
+        sesshdrs = {'User-Agent': self.getUserAgent()}
+        async with aiohttp.ClientSession(connector=connector, timeout=atimeout, headers=sesshdrs) as sess:
             try:
                 await self._reqHas(sha256)
                 async with sess.request(method, url, headers=headers, params=params, ssl=ssl,
@@ -1811,7 +1813,7 @@ class Axon(s_cell.Cell):
             proxy (str|bool): The proxy value.
 
         Notes:
-            The response body will be stored, regardless of the response code. The ``ok`` value in the response does not
+            The response body will be stored, regardless of the response code. The `ok` value in the response does not
             reflect that a status code, such as a 404, was encountered when retrieving the URL.
 
             The ssl dictionary may contain the following values::
@@ -1866,7 +1868,8 @@ class Axon(s_cell.Cell):
 
         atimeout = aiohttp.ClientTimeout(total=timeout)
 
-        async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess:
+        sesshdrs = {'User-Agent': self.getUserAgent()}
+        async with aiohttp.ClientSession(connector=connector, timeout=atimeout, headers=sesshdrs) as sess:
 
             try:
                 async with sess.request(method, url, headers=headers, params=params, json=json, data=body, ssl=ssl,

{synapse-3.1.0 → synapse-3.3.0}/synapse/common.py

@@ -277,7 +277,7 @@ def genpath(*paths):
     '''
     Return an absolute path of the joining of the arguments as path elements
 
-    Performs home directory(``~``) and environment variable expansion on the joined path
+    Performs home directory(`~`) and environment variable expansion on the joined path
 
     Args:
         *paths ([str,...]): A list of path elements
@@ -371,12 +371,12 @@ def genfile(*paths):
         *paths: A list of paths to join together to make the file.
 
     Notes:
-        If the file already exists, the fd returned is opened in ``r+b`` mode.
-        Otherwise, the fd is opened in ``w+b`` mode.
+        If the file already exists, the fd returned is opened in `r+b` mode.
+        Otherwise, the fd is opened in `w+b` mode.
 
         The file position is set to the start of the file.  The user is
-        responsible for truncating (``fd.truncate()``) if the existing file
-        contents are not desired, or seeking to the end (``fd.seek(0, 2)``)
+        responsible for truncating (`fd.truncate()`) if the existing file
+        contents are not desired, or seeking to the end (`fd.seek(0, 2)`)
         to append.
 
     Returns:
@@ -461,7 +461,7 @@ def getDirSize(*paths):
 
     Returns:
         tuple: Tuple of total real and total apparent size of all normal files and directories underneath
-        ``*paths`` plus ``*paths`` itself.
+        `*paths` plus `*paths` itself.
     '''
     def getsize(path):
         try:
@@ -659,8 +659,8 @@ def spin(genr):
         genr: Any generator or iterable valu.
 
     Notes:
-        This generator is exhausted via the ``collections.dequeue()``
-        constructor with a ``maxlen=0``, which will quickly exhaust an
+        This generator is exhausted via the `collections.dequeue()`
+        constructor with a `maxlen=0`, which will quickly exhaust an
         iterator staying in C code as much as possible.
 
     Returns:

{synapse-3.1.0 → synapse-3.3.0}/synapse/cortex.py

@@ -46,6 +46,7 @@ import synapse.lib.version as s_version
 import synapse.lib.hashitem as s_hashitem
 import synapse.lib.jsonstor as s_jsonstor
 import synapse.lib.modelrev as s_modelrev
+import synapse.lib.nodefuse as s_nodefuse
 import synapse.lib.stormsvc as s_stormsvc
 import synapse.lib.lmdbslab as s_lmdbslab
 
@@ -1116,6 +1117,8 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
              'desc': 'Controls the ability to check if the Axon contains a file.'},
             {'perm': ('axon', 'del'), 'gate': 'cortex',
              'desc': 'Controls the ability to remove a file from the Axon.'},
+            {'perm': ('axon', 'wput'), 'gate': 'cortex',
+             'desc': 'Controls the ability to push a file from the Axon to a URL.'},
 
             {'perm': ('layer',), 'gate': 'cortex',
              'desc': 'Controls all layer permissions.'},
@@ -2062,6 +2065,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             ctor.pkgname = pkgdef.get('name')
 
         ctor.deprecated = cdef.get('deprecated')
+        ctor._cortex_edition = cdef.get('_edition')
 
         def getRuntPode(model):
             props = {
@@ -2071,6 +2075,9 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             if ctor.pkgname:
                 props['package'] = ctor.pkgname
 
+            if ctor._cortex_edition is not None:
+                props['edition'] = ctor._cortex_edition
+
             if ctor.deprecated:
                 props['deprecated'] = True
 
@@ -2573,6 +2580,9 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             self._initEasyPerm(gdef)
             self.pkggraphs[gdef['iden']] = gdef
 
+        extra = self.getLogExtra(pkg=name, vers=pkgvers)
+        logger.info(f'Loaded Storm package: {name}@{pkgvers}.', extra=extra)
+
     def _runStormPkgOnload(self, pkgdef):
         name = pkgdef.get('name')
         inits = pkgdef.get('inits')
@@ -2689,6 +2699,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             self._popStormCmd(name)
 
         pkgname = pkgdef.get('name')
+        pkgvers = pkgdef.get('version')
 
         # match on the provenance we derived at load time rather than re-deriving
         # the idens, so this stays correct regardless of the package def contents.
@@ -2697,6 +2708,9 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
 
         self.stormpkgs.pop(pkgname, None)
 
+        extra = self.getLogExtra(pkg=pkgname, vers=pkgvers)
+        logger.info(f'Unloaded Storm package: {pkgname}@{pkgvers}.', extra=extra)
+
     def getStormSvc(self, name):
         return self.svcs.get(name)
 
@@ -3051,7 +3065,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         '''
         While active, discover storm services registered with AHA and add
         previously unknown ones to the cortex automatically. Uses the initial
-        AHA service listing and tracks ``svc:add`` topology updates.
+        AHA service listing and tracks `svc:add` topology updates.
         '''
         async with await s_base.Base.anit() as base:
 
@@ -5898,7 +5912,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
 
     async def _reqDecEncryption(self, encryption):
         '''
-        Return an encryption dict whose ``seed`` is the plaintext pbkdf2 seed. A
+        Return an encryption dict whose `seed` is the plaintext pbkdf2 seed. A
         per-deployment package stores its seed RSA-encrypted to our deployment key,
         so decrypt it (cached) with the deployment private key.
         '''
@@ -6061,7 +6075,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
         except Exception as e:
             return s_common.retnexc(e)
 
-    def _logStormQuery(self, text, user, info=None, meta=None):
+    def _logStormQuery(self, text, user, info=None, meta=None, via=None):
         '''
         Log a storm query.
 
@@ -6070,6 +6084,7 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             user: The User who ran the query.
             info (dict): Fields to record alongside the query.
             meta (dict): The caller supplied meta opt, if any.
+            via (str): The name of the Storm command or library which dispatched the query.
         '''
         if self.stormlog:
             if info is None:
@@ -6083,8 +6098,12 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             if meta is not None:
                 info['meta'] = meta
 
-            stormlogger.log(self.stormloglvl, 'Executing storm query {%s} as [%s]', text, user.name,
-                            extra=self.getLogExtra(**info))
+            if via is None:
+                stormlogger.log(self.stormloglvl, 'Executing storm query {%s} as [%s]', text, user.name,
+                                extra=self.getLogExtra(**info))
+            else:
+                stormlogger.log(self.stormloglvl, 'Executing storm query via %s {%s} as [%s]', via, text, user.name,
+                                extra=self.getLogExtra(**info))
 
     async def getCoreInfoV2(self):
         return {
@@ -6112,6 +6131,9 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             if cmd.pkgname:
                 entry['package'] = cmd.pkgname
 
+            if (edition := cmd._cortex_edition) is not None:
+                entry['edition'] = edition
+
             cmds.append(entry)
 
         ret = {
@@ -6469,6 +6491,64 @@ class Cortex(s_oauth.OAuthMixin, s_axon.HasAxon, s_jsonstor.HasJsonStor, s_cell.
             yield
             self.migration = False
 
+    async def fuseNodes(self, srcndef, dstndef, useriden):
+        '''
+        Fuse the src node into the dst node across every layer in the Cortex.
+
+        Node fusion writes to arbitrary layers rather than a single view's write layer, and
+        the edits it makes depend on the state it reads. The edits are computed here, before
+        any of them are applied, and are then handed to each layer with
+        Layer.saveNodeEdits(). That layer resolves them against its own current state and
+        puts the resolved edits in the nexus log, so a mirror applies exactly the edits which
+        were computed here rather than recomputing them from its own state.
+
+        A fuse of a heavily referenced node can need a very large number of edits, so each
+        layer's edits are applied in chunks of one nexus operation each rather than all at
+        once. A fuse is therefore not transactional, and an interruption can leave part of it
+        applied. See NodeFuser._iterNodeEdits() for the ordering which makes that recoverable.
+
+        The reads are not serialized against other writes, so a write to src which lands after
+        the edits are computed is not accounted for and is not detected. Running a fuse during
+        a maintenance window, as recommended in the Storm API docs, avoids this. Executing the
+        fuse as requested is this method's responsibility; a concurrent edit landing on src is
+        outside that scope.
+
+        Args:
+            srcndef (tuple): The (form, valu) of the node to fuse from. It will be deleted.
+            dstndef (tuple): The (form, valu) of the node to fuse into. It will be kept.
+            useriden (str): The iden of the user running the fuse.
+
+        Returns:
+            dict: The warnings to emit and the layers which failed.
+        '''
+        # tick is sampled once and carried in the nodeedit meta, so that the leader and every
+        # mirror record the same time in the layer node edit logs.
+        meta = {'time': s_common.now(), 'user': useriden}
+
+        # A fuse is a global admin operation which rewrites data across every layer in the
+        # Cortex and is not transactional, so both ends are logged: an interruption leaves a
+        # start line with no completion line, which is what says a re-run may be needed.
+        logger.info(f'fuseNodes() starting fuse of {srcndef[0]}={srcndef[1]!r} into '
+                    f'{dstndef[0]}={dstndef[1]!r}')
+
+        # the fuser owns the spooled state the edits are accumulated in, so it stays open
+        # until they have all been applied
+        async with await s_nodefuse.NodeFuser.anit(self, useriden) as fuser:
+
+            await fuser.getLayerEdits(srcndef, dstndef)
+
+            await fuser.applyLayerEdits(meta)
+
+            # returned even with nothing to apply, since discovery may have produced
+            # warnings of its own which the caller still needs to emit
+            result = fuser.getResult()
+
+            logger.info(f'fuseNodes() completed fuse of {srcndef[0]}={srcndef[1]!r} into '
+                        f'{dstndef[0]}={dstndef[1]!r} '
+                        f'(renames: {len(fuser.ndefmap)}, failed layers: {len(result["failed"])})')
+
+            return result
+
     async def iterFormRows(self, layriden, form, stortype=None, startvalu=None):
         '''
         Yields nid, valu tuples of nodes of a single form, optionally (re)starting at startvalu.

{synapse-3.1.0 → synapse-3.3.0}/synapse/daemon.py

@@ -48,10 +48,10 @@ class Sess(s_base.Base):
 
 async def t2call(link, meth, args, kwargs, first=True):
     '''
-    Call the given ``meth(*args, **kwargs)`` and handle the response to provide
+    Call the given `meth(*args, **kwargs)` and handle the response to provide
     telepath task v2 events to the given link.
 
-    The ``first`` argument may be set to ``False`` to skip sending an initial ``t2:genr``
+    The `first` argument may be set to `False` to skip sending an initial `t2:genr`
     message when using a using a link which has already been initialized (such as when sending
     a link to a spawned process).
     '''

{synapse-3.1.0 → synapse-3.3.0}/synapse/data/lark/storm.lark

@@ -47,7 +47,8 @@ _editblock: "[" _editoper* "]"
 // A single edit operation
 _editoper: editnodeadd
             | editpropset | edittagpropset | edittagadd | editcondpropset
-            | editvirtpropset | edittagvirtset | edittagpropvirtset
+            | editvirtpropset | editvirtcondpropset | edittagvirtset | edittagvirtcondset
+            | edittagpropvirtset
             | editpropsetmulti | editpropdel | edittagpropdel | edittagdel
             | editparens | edgeaddn1 | edgedeln1 | edgeaddn2 | edgedeln2
 
@@ -59,10 +60,14 @@ edittagadd: "+" tagname [(EQSPACE | EQNOSPACE | TRYSET) _valu]
 edittagvirtset: "+" tagnamevirt virtprop (EQSPACE | EQNOSPACE | TRYSET) _valu
               | "+" "?" tagnamevirt virtprop (EQSPACE | EQNOSPACE | TRYSET) _valu -> edittagvirttryset
 
+edittagvirtcondset: "+" tagnamevirt virtprop condsetoper _valu
+                  | "+" "?" tagnamevirt virtprop condsetoper _valu -> edittagvirtcondtryset
+
 edittagdel: EXPRMINUS tagname
 editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
 editcondpropset: relprop condsetoper _valu
 editvirtpropset: relprop virtprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
+editvirtcondpropset: relprop virtprop condsetoper _valu
 editpropsetmulti: relprop (MODSETMULTI | TRYMODSETMULTI) _valu
 editpropdel: EXPRMINUS relprop
 editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYMODSET) _valu
@@ -77,9 +82,11 @@ MODSETMULTI.5: "++=" | "--="
 TRYMODSETMULTI.1: "?++=" | "?--="
 TRYSET.1: "?="
 
-condsetoper: ("*" UNSET | _DEREF "$" _condvarvaluatom) "="
-    | ("*" UNSET | _DEREF "$" _condvarvaluatom) "?=" -> condtrysetoper
+condsetoper: ("*" (UNSET | CONDMIN | CONDMAX) | _DEREF "$" _condvarvaluatom) "="
+    | ("*" (UNSET | CONDMIN | CONDMAX) | _DEREF "$" _condvarvaluatom) "?=" -> condtrysetoper
 UNSET: "unset"
+CONDMIN: "min"
+CONDMAX: "max"
 _condvarvaluatom: condvarvalue | condvarderef | condfunccall
 condvarvalue: VARTOKN -> varvalue
 

{synapse-3.1.0 → synapse-3.3.0}/synapse/data/skills/storm/SKILL.md

@@ -171,6 +171,10 @@ Comparison operators: `=`, `!=`, `<`, `>`, `<=`, `>=`, `~=` (regex), `^=` (prefi
 [ -:asn ]                                     // delete property
 [ :prop ?= value ]                            // try-set (ignore type errors)
 [ :prop*unset= value ]                        // set only if unset (conditional)
+[ :prop*min= value ]                          // keep the lesser of the current and given value
+[ :prop*max= value ]                          // keep the greater (widens rather than overwrites)
+[ :seen.min*min?= $first ]                     // conditional set on a virt, with the try oper
+[ +?#(tag.name).max*max= $updated ]            // and on a tag timestamp (tag name parenthesized)
 [ :prop={ foo:bar } ]                         // assign from a subquery (yields a node; its value is assigned)
 [ :prop={[ foo:bar=baz ]} ]                   // the subquery may create the node to assign
 
@@ -197,6 +201,45 @@ Comparison operators: `=`, `!=`, `<`, `>`, `<=`, `>=`, `~=` (regex), `^=` (prefi
 )]
 ```
 
+**Setting an array property** distinguishes "absent" from "present but empty", so pick the
+form deliberately:
+
+```storm
+[ :cookies = $vals ]                          // a list; an EMPTY list stores an empty array (), NOT unset
+[ :cookies={ ... } ]                          // a subquery yielding nothing also stores ()
+[ :cookies ?= (null) ]                        // ?= with (null) skips the set, leaving the prop unset
+[ :cookies = (null) ]                         // plain = with (null) raises BadTypeValu
+[ :cookies = "a=1; b=2" ]                     // WRONG: a delimited string is not split; the set fails
+```
+
+Set the **empty array** when the source data carried the field but none of its values were
+usable -- it is a positive statement that nothing valid was there -- and leave the property
+**unset** when the field was absent entirely.
+
+An array property's `size` virt keeps both cases cheap to lift, so an always-set property
+does not cost you the "which nodes actually have values?" query (`=`, `<`, `>`, `<=`, `>=`
+and `range=` are all supported):
+
+```storm
+inet:http:response:cookies                    // set at all, empty array included
+inet:http:response:cookies.size>0             // has at least one value
+inet:http:response:cookies.size=0             // present, but nothing valid was there
+inet:http:response +:cookies.size>0           // same, as a filter
+```
+
+A type that splits one delimited string into several values (e.g. `inet:http:cookie` on
+`;`) only does so while **constructing nodes**, never while norming an array property
+value. So build the values in a subquery, or pass a list:
+
+```storm
+[ :cookies={[ inet:http:cookie="a=1; b=2" ]} ]   // two cookie nodes, both assigned
+[ :cookies = ("a=1", "b=2") ]                    // equivalent, as a list
+```
+
+An element the type rejects is skipped when the values come from node construction, but one
+bad element fails the whole set when they come from a list -- `?=` guards the property, not
+the individual items.
+
 ### Variables & Expressions
 
 ```storm
@@ -416,6 +459,13 @@ else { ... }
 for $item in $list { ... }
 for ($key, $val) in $dict { ... }
 
+// CAREFUL: with a node inbound, a loop yields that node on EVERY iteration, so the
+// statements after the loop start running before the loop has finished. Usually the fix
+// is to run the loop in a subquery, whose output is discarded:
+{ for $item in $list { ... } }
+// Alternatively build the variable up on an empty pipeline (before any `yield`), or
+// defer whatever follows with `fini { }`.
+
 // While loop
 while $condition { ... }
 
@@ -704,6 +754,12 @@ $x = (true)
 $y = (999)
 $dict = ({})
 $list = ([])
+
+// WRONG - redundant parens inside a JSON expression
+$props = ({"ids": ([$iden]), "opts": ({"asof": $asof})})
+
+// CORRECT
+$props = ({"ids": [$iden], "opts": {"asof": $asof}})
 ```
 
 ### 3. Forgetting `|` After Inline Commands

{synapse-3.1.0 → synapse-3.3.0}/synapse/datamodel.py

@@ -126,6 +126,10 @@ class Prop:
         self.locked = False
         self.deprecated = self.info.get('deprecated', False)
 
+        # typedef has already been poly converted by Model.processPropdefs(), so self.type is
+        # a Poly for a non-array prop and an Array whose arraytype is a Poly for an array
+        # one. self.type.ispoly is False for an array prop - the poly is on the element type,
+        # not on the Array. See processPropdefs().
         self.type = self.modl.getPropTypeClone(typedef, self.info)
         self.typehash = self.type.typehash
 
@@ -673,6 +677,7 @@ class Model:
         self.formprefixcache = s_cache.LruDict(PREFIX_CACHE_SIZE)
 
         self._lookup_hints = None
+        self._virts_by_type = None
 
         self._type_pends = collections.defaultdict(list)
         self._modeldef = {
@@ -744,6 +749,33 @@ class Model:
             return ()
         return list(props.values())
 
+    def getVirtsByType(self, name):
+        '''
+        Return the (formname, virtname) tuples for the indexed virtual properties
+        of a form primary property which reference the given form name. A virt on
+        a secondary property is not a ref because the node for the property value
+        carries the ref, the same as Form.getRefsOut().
+        '''
+        if self._virts_by_type is None:
+
+            virtsbytype = collections.defaultdict(list)
+
+            for formname, form in self.forms.items():
+                for virtname, virtinfo in form.type.virts.items():
+
+                    typename = virtinfo[0].name
+                    if self.forms.get(typename) is None:
+                        continue
+
+                    if form.type.virtindx.get(virtname) is None:
+                        continue
+
+                    virtsbytype[typename].append((formname, virtname))
+
+            self._virts_by_type = virtsbytype
+
+        return self._virts_by_type.get(name, ())
+
     def getProps(self):
         return [pobj for pname, pobj in self.props.items()
                 if not (isinstance(pname, tuple))]
@@ -1244,6 +1276,20 @@ class Model:
         return polyinfo
 
     def convertTypedef(self, typedef):
+        '''
+        Return the given prop typedef in poly form.
+
+        Everything comes back as a ('poly', {...}) typedef: a tuple of constituents is
+        converted, and anything else is wrapped as a single member poly. A typedef which
+        already names a Poly or an Array type is the one case returned as it stands, since it
+        is in that form already.
+
+        Note that converting a typedef here does not make the resulting Prop.type a Poly. An
+        array prop declares its *element* type here and its container opts under the 'array'
+        prop info key, so getPropTypeClone() wraps what this returns in an Array - leaving
+        Prop.type an Array with ispoly False and the poly on Prop.type.arraytype. See
+        processPropdefs().
+        '''
         typename = typedef[0]
 
         if not isinstance(typename, tuple):
@@ -1268,14 +1314,14 @@ class Model:
         '''
         Enforce where inline type opts may be declared on a prop.
 
-        A prop type is declared ``(typename, opts)`` and must reference a named type with empty
+        A prop type is declared `(typename, opts)` and must reference a named type with empty
         opts (or, for a poly prop, a tuple of named constituents). A prop that needs custom
-        normalization opts (``regex``, ``enums``, ``names``, ``precision``, ...) must be declared
+        normalization opts (`regex`, `enums`, `names`, `precision`, ...) must be declared
         once as a named type and referenced by name.
 
         An array prop declares its element type in the typedef slot and its container opts
-        (``uniq``/``sorted``/``split``) under the ``array`` prop info key. The legacy
-        ``('array', {...})`` container typedef is no longer supported.
+        (`uniq`/`sorted`/`split`) under the `array` prop info key. The legacy
+        `('array', {...})` container typedef is no longer supported.
         '''
         prefix = f'{formname}:{propname}' if formname is not None else propname
 
@@ -1324,7 +1370,36 @@ class Model:
             raise s_exc.BadPropDef(mesg=mesg, name=propname)
 
     def processPropdefs(self, propdefs, formname=None):
+        '''
+        Validate and poly convert a sequence of prop definitions.
+
+        Every prop typedef is converted to poly form here via convertTypedef(), whatever it
+        was declared as - so by the time Prop.__init__() calls getPropTypeClone(), the
+        declared typedef is always ('poly', {...}). Two consequences are worth stating
+        outright, because the conversion sits two indirections away from where a reader
+        looking at Prop.__init__() or at a Type subclass would find it:
+
+        * Every property's *value* type is a Poly, but for an array prop that Poly is
+          Prop.type.arraytype rather than Prop.type itself. getPropTypeClone() wraps the
+          converted element type in an Array, and Array.postTypeInit() clones the poly onto
+          arraytype, so Prop.type is a Poly for a non-array prop and an Array for an array
+          one. Prop.type.ispoly is therefore True for exactly the non-array props and False
+          for every array prop - `if not prop.type.ispoly` is a live test which selects the
+          array props (see getTypeForms() and getStepForms()), not a dead branch. Code
+          reaching for a property's member type wants `prop.type.arraytype if
+          prop.type.isarray else prop.type`.
+
+        * A stored property value is always a typed value - a (typename, valu) tuple - and
+          not a bare one, including each element of an array. Re-normalizing a stored value
+          goes through Type.normFromTypedValu() rather than Type.norm().
 
+        Args:
+            propdefs (list): The (name, typedef, propinfo) triples to process.
+            formname (str): The name of the form these props belong to, if any.
+
+        Returns:
+            tuple: The processed (name, typedef, propinfo) triples.
+        '''
         realdefs = []
 
         for pname, typedef, propinfo in propdefs:
@@ -1743,6 +1818,7 @@ class Model:
         self.formprefixcache.clear()
         self.pivchaincache.clear()
         self._lookup_hints = None
+        self._virts_by_type = None
 
         return form
 
@@ -1857,6 +1933,7 @@ class Model:
         self.formprefixcache.clear()
         self.pivchaincache.clear()
         self._lookup_hints = None
+        self._virts_by_type = None
 
         if parentform:
             self.childforms[parentform.name].remove(formname)

{synapse-3.1.0 → synapse-3.3.0}/synapse/exc.py

@@ -253,7 +253,7 @@ class JsonRpcError(SynErr):
     '''
     A JSON-RPC error carrying an application-defined code (and optional data).
 
-    This should contain a numeric ``code`` and a ``mesg``. An optional ``data`` value
+    This should contain a numeric `code` and a `mesg`. An optional `data` value
     may be provided to convey additional structured information to the caller.
     '''
     @classmethod