synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/lib/trigger.py

@@ -21,6 +21,8 @@ Conditions = set((
     'node:add',
     'node:del',
     'prop:set',
+    'edge:add',
+    'edge:del'
 ))
 
 RecursionDepth = contextvars.ContextVar('RecursionDepth', default=0)
@@ -37,11 +39,13 @@ TrigSchema = {
         'user': {'type': 'string', 'pattern': s_config.re_iden},
         'view': {'type': 'string', 'pattern': s_config.re_iden},
         'form': {'type': 'string', 'pattern': _formre},
+        'n2form': {'type': 'string', 'pattern': _formre},
         'tag': {'type': 'string', 'pattern': _tagre},
         'prop': {'type': 'string', 'pattern': _propre},
+        'verb': {'type': 'string', },
         'name': {'type': 'string', },
         'doc': {'type': 'string', },
-        'cond': {'enum': ['node:add', 'node:del', 'tag:add', 'tag:del', 'prop:set']},
+        'cond': {'enum': ['node:add', 'node:del', 'tag:add', 'tag:del', 'prop:set', 'edge:add', 'edge:del']},
         'storm': {'type': 'string'},
         'async': {'type': 'boolean'},
         'enabled': {'type': 'boolean'},
@@ -69,6 +73,14 @@ TrigSchema = {
             'if': {'properties': {'cond': {'const': 'prop:set'}}},
             'then': {'required': ['prop']},
         },
+        {
+            'if': {'properties': {'cond': {'const': 'edge:add'}}},
+            'then': {'required': ['verb']},
+        },
+        {
+            'if': {'properties': {'cond': {'const': 'edge:del'}}},
+            'then': {'required': ['verb']},
+        },
     ],
 }
 TrigSchemaValidator = s_config.getJsValidator(TrigSchema)
@@ -89,9 +101,9 @@ class Triggers:
         self.view = view
         self.triggers = {}
 
-        self.tagadd = collections.defaultdict(list)    # (form, tag): [ Triger ... ]
-        self.tagset = collections.defaultdict(list)    # (form, tag): [ Triger ... ]
-        self.tagdel = collections.defaultdict(list)    # (form, tag): [ Triger ... ]
+        self.tagadd = collections.defaultdict(list)    # (form, tag): [ Trigger ... ]
+        self.tagset = collections.defaultdict(list)    # (form, tag): [ Trigger ... ]
+        self.tagdel = collections.defaultdict(list)    # (form, tag): [ Trigger ... ]
 
         self.tagaddglobs = collections.defaultdict(s_cache.TagGlobs)    # form: TagGlobs
         self.tagsetglobs = collections.defaultdict(s_cache.TagGlobs)    # form: TagGlobs
@@ -101,6 +113,15 @@ class Triggers:
         self.nodedel = collections.defaultdict(list)   # form: [ Trigger ... ]
         self.propset = collections.defaultdict(list)   # prop: [ Trigger ... ]
 
+        self.edgeadd = collections.defaultdict(list)   # (n1form, verb, n2form: [ Trigger ... ]
+        self.edgedel = collections.defaultdict(list)   # (n1form, verb, n2form: [ Trigger ... ]
+
+        self.edgeaddglobs = collections.defaultdict(s_cache.EdgeGlobs)  # (n1form, n2form: [ EdgeGlobs ... ]
+        self.edgedelglobs = collections.defaultdict(s_cache.EdgeGlobs)  # (n1form, n2form: [ EdgeGlobs ... ]
+
+        self.edgeaddcache = s_cache.LruDict()
+        self.edgedelcache = s_cache.LruDict()
+
     @contextlib.contextmanager
     def _recursion_check(self):
 
@@ -182,6 +203,106 @@ class Triggers:
                 for _, trig in globs.get(tag):
                     await trig.execute(node, vars=vars, view=view)
 
+    async def runEdgeAdd(self, n1, verb, n2, view=None):
+        n1form = n1.form.name if n1 else None
+        n2form = n2.form.name if n2 else None
+        n2iden = n2.iden() if n2 else None
+        varz = {'auto': {'opts': {'verb': verb, 'n2iden': n2iden}}}
+        with self._recursion_check():
+            cachekey = (n1form, verb, n2form)
+            cached = self.edgeaddcache.get(cachekey)
+            if cached is None:
+                cached = []
+                for trig in self.edgeadd.get((None, verb, None), ()):
+                    cached.append(trig)
+
+                globs = self.edgeaddglobs.get((None, None))
+                if globs:
+                    for _, trig in globs.get(verb):
+                        cached.append(trig)
+
+                if n1:
+                    for trig in self.edgeadd.get((n1form, verb, None), ()):
+                        cached.append(trig)
+
+                    globs = self.edgeaddglobs.get((n1form, None))
+                    if globs:
+                        for _, trig in globs.get(verb):
+                            cached.append(trig)
+
+                if n2:
+                    for trig in self.edgeadd.get((None, verb, n2form), ()):
+                        cached.append(trig)
+
+                    globs = self.edgeaddglobs.get((None, n2form))
+                    if globs:
+                        for _, trig in globs.get(verb):
+                            cached.append(trig)
+
+                if n1 and n2:
+                    for trig in self.edgeadd.get((n1form, verb, n2form), ()):
+                        cached.append(trig)
+
+                    globs = self.edgeaddglobs.get((n1form, n2form))
+                    if globs:
+                        for _, trig in globs.get(verb):
+                            cached.append(trig)
+
+                self.edgeaddcache[cachekey] = cached
+
+            for trig in cached:
+                await trig.execute(n1, vars=varz, view=view)
+
+    async def runEdgeDel(self, n1, verb, n2, view=None):
+        n1form = n1.form.name if n1 else None
+        n2form = n2.form.name if n2 else None
+        n2iden = n2.iden() if n2 else None
+        varz = {'auto': {'opts': {'verb': verb, 'n2iden': n2iden}}}
+        with self._recursion_check():
+            cachekey = (n1form, verb, n2form)
+            cached = self.edgedelcache.get(cachekey)
+            if cached is None:
+                cached = []
+                for trig in self.edgedel.get((None, verb, None), ()):
+                    cached.append(trig)
+
+                globs = self.edgedelglobs.get((None, None))
+                if globs:
+                    for _, trig in globs.get(verb):
+                        cached.append(trig)
+
+                if n1:
+                    for trig in self.edgedel.get((n1form, verb, None), ()):
+                        cached.append(trig)
+
+                    globs = self.edgedelglobs.get((n1form, None))
+                    if globs:
+                        for _, trig in globs.get(verb):
+                            cached.append(trig)
+
+                if n2:
+                    for trig in self.edgedel.get((None, verb, n2form), ()):
+                        cached.append(trig)
+
+                    globs = self.edgedelglobs.get((None, n2form))
+                    if globs:
+                        for _, trig in globs.get(verb):
+                            cached.append(trig)
+
+                if n1 and n2:
+                    for trig in self.edgedel.get((n1form, verb, n2form), ()):
+                        cached.append(trig)
+
+                    globs = self.edgedelglobs.get((n1form, n2form))
+                    if globs:
+                        for _, trig in globs.get(verb):
+                            cached.append(trig)
+
+                self.edgedelcache[cachekey] = cached
+
+            for trig in cached:
+                await trig.execute(n1, vars=varz, view=view)
+
     async def load(self, tdef):
 
         trig = Trigger(self.view, tdef)
@@ -194,6 +315,8 @@ class Triggers:
         tag = trig.tdef.get('tag')
         form = trig.tdef.get('form')
         prop = trig.tdef.get('prop')
+        verb = trig.tdef.get('verb')
+        n2form = trig.tdef.get('n2form')
 
         if cond not in Conditions:
             raise s_exc.NoSuchCond(name=cond)
@@ -206,6 +329,9 @@ class Triggers:
             if tag is None:
                 raise s_exc.BadOptValu(mesg='missing tag')
             s_chop.validateTagMatch(tag)
+        if cond in ('edge:add', 'edge:del') and verb is None:
+            raise s_exc.BadOptValu(mesg='verb must be present for edge:add or edge:del')
+
         if prop is not None and cond != 'prop:set':
             raise s_exc.BadOptValu(mesg='prop parameter invalid')
 
@@ -237,6 +363,20 @@ class Triggers:
             else:
                 self.tagdelglobs[form].add(tag, trig)
 
+        elif cond == 'edge:add':
+            self.edgeaddcache.clear()
+            if '*' not in verb:
+                self.edgeadd[(form, verb, n2form)].append(trig)
+            else:
+                self.edgeaddglobs[(form, n2form)].add(verb, trig)
+
+        elif cond == 'edge:del':
+            self.edgedelcache.clear()
+            if '*' not in verb:
+                self.edgedel[(form, verb, n2form)].append(trig)
+            else:
+                self.edgedelglobs[(form, n2form)].add(verb, trig)
+
         self.triggers[trig.iden] = trig
         return trig
 
@@ -292,6 +432,34 @@ class Triggers:
             globs.rem(tag, trig)
             return trig
 
+        if cond == 'edge:add':
+            verb = trig.tdef['verb']
+            form = trig.tdef.get('form')
+            n2form = trig.tdef.get('n2form')
+
+            self.edgeaddcache.clear()
+            if '*' not in verb:
+                self.edgeadd[(form, verb, n2form)].remove(trig)
+                return trig
+
+            globs = self.edgeaddglobs.get((form, n2form))
+            globs.rem(verb, trig)
+            return trig
+
+        if cond == 'edge:del':
+            verb = trig.tdef['verb']
+            form = trig.tdef.get('form')
+            n2form = trig.tdef.get('n2form')
+
+            self.edgedelcache.clear()
+            if '*' not in verb:
+                self.edgedel[(form, verb, n2form)].remove(trig)
+                return trig
+
+            globs = self.edgedelglobs.get((form, n2form))
+            globs.rem(verb, trig)
+            return trig
+
         raise AssertionError('trigger has invalid condition')
 
     def get(self, iden):
@@ -440,6 +608,14 @@ class Trigger:
         if prop is not None:
             props['prop'] = prop
 
+        verb = self.tdef.get('verb')
+        if verb is not None:
+            props['verb'] = verb
+
+        n2form = self.tdef.get('n2form')
+        if n2form is not None:
+            props['n2form'] = n2form
+
         pnorms = {}
         for prop, valu in props.items():
             formprop = form.props.get(prop)

synapse/lib/types.py

@@ -239,7 +239,7 @@ class Type:
         return cmpr
 
     def _ctorCmprRe(self, text):
-        regx = regex.compile(text)
+        regx = regex.compile(text, flags=regex.I)
 
         def cmpr(valu):
             vtxt = self.repr(valu)

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 152, 0)
+version = (2, 154, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'f93d8dfc6fd1c4a1ed735ec746100100a9190e2e'
+commit = '6c9600992a0ca2563239bc86b6631bd599707e3f'

synapse/lib/view.py

@@ -746,6 +746,7 @@ class View(s_nexus.Pusher):  # type: ignore
 
         await self.info.set('layers', [lyr.iden for lyr in self.layers])
         await self.core.feedBeholder('view:addlayer', {'iden': self.iden, 'layer': layriden, 'indx': indx}, gates=[self.iden, layriden])
+        self.core._calcViewsByLayer()
 
     @s_nexus.Pusher.onPushAuto('view:setlayers')
     async def setLayers(self, layers):
@@ -753,14 +754,11 @@ class View(s_nexus.Pusher):  # type: ignore
         Set the view layers from a list of idens.
         NOTE: view layers are stored "top down" (the write layer is self.layers[0])
         '''
-        for view in self.core.views.values():
-            if view.parent is self:
-                raise s_exc.ReadOnlyLayer(mesg='May not change layers that have been forked from')
+        layrs = []
 
         if self.parent is not None:
-            raise s_exc.ReadOnlyLayer(mesg='May not change layers of forked view')
-
-        layrs = []
+            mesg = 'You cannot set the layers of a forked view.'
+            raise s_exc.BadArg(mesg=mesg)
 
         for iden in layers:
             layr = self.core.layers.get(iden)
@@ -777,6 +775,39 @@ class View(s_nexus.Pusher):  # type: ignore
         await self.info.set('layers', layers)
         await self.core.feedBeholder('view:setlayers', {'iden': self.iden, 'layers': layers}, gates=[self.iden, layers[0]])
 
+        await self._calcChildViews()
+        self.core._calcViewsByLayer()
+
+    async def _calcChildViews(self):
+
+        todo = collections.deque([self])
+
+        byparent = collections.defaultdict(list)
+        for view in self.core.views.values():
+            if view.parent is None:
+                continue
+
+            byparent[view.parent].append(view)
+
+        while todo:
+
+            view = todo.pop()
+
+            for child in byparent.get(view, ()):
+
+                layers = [child.layers[0]]
+                layers.extend(view.layers)
+
+                child.layers = layers
+
+                # convert layers to a list of idens...
+                lids = [layr.iden for layr in layers]
+                await child.info.set('layers', lids)
+
+                await self.core.feedBeholder('view:setlayers', {'iden': child.iden, 'layers': lids}, gates=[child.iden, lids[0]])
+
+                todo.append(child)
+
     async def fork(self, ldef=None, vdef=None):
         '''
         Make a new view inheriting from this view with the same layers and a new write layer on top
@@ -961,6 +992,24 @@ class View(s_nexus.Pusher):  # type: ignore
 
         await self.triggers.runPropSet(node, prop, oldv, view=view)
 
+    async def runEdgeAdd(self, n1, edge, n2, view=None):
+        if not n1.snap.trigson:
+            return
+
+        if view is None:
+            view = self.iden
+
+        await self.triggers.runEdgeAdd(n1, edge, n2, view=view)
+
+    async def runEdgeDel(self, n1, edge, n2, view=None):
+        if not n1.snap.trigson:
+            return
+
+        if view is None:
+            view = self.iden
+
+        await self.triggers.runEdgeDel(n1, edge, n2, view=view)
+
     async def addTrigger(self, tdef):
         '''
         Adds a trigger to the view.

synapse/models/inet.py

@@ -28,6 +28,8 @@ udots = regex.compile(r'[\u3002\uff0e\uff61]')
 cidrmasks = [((0xffffffff - (2 ** (32 - i) - 1)), (2 ** (32 - i))) for i in range(33)]
 ipv4max = 2 ** 32 - 1
 
+rfc6598 = ipaddress.IPv4Network('100.64.0.0/10')
+
 def getAddrType(ip):
 
     if ip.is_multicast:
@@ -45,6 +47,9 @@ def getAddrType(ip):
     if ip.is_reserved:
         return 'reserved'
 
+    if ip in rfc6598:
+        return 'shared'
+
     return 'unicast'
 
 class Addr(s_types.Str):
@@ -924,6 +929,11 @@ class Url(s_types.Str):
                 except Exception:
                     pass
 
+            # allow MSFT specific wild card syntax
+            # https://learn.microsoft.com/en-us/windows/win32/http/urlprefix-strings
+            if host is None and part == '+':
+                host = '+'
+
         if host and local:
             raise s_exc.BadTypeValu(valu=orig, name=self.name,
                                     mesg='Host specified on local-only file URI') from None
@@ -950,10 +960,12 @@ class Url(s_types.Str):
             if port is not None:
                 hostparts = f'{hostparts}:{port}'
 
-        if proto != 'file':
-            if (not subs.get('fqdn')) and (subs.get('ipv4') is None) and (subs.get('ipv6') is None):
-                raise s_exc.BadTypeValu(valu=orig, name=self.name,
-                                        mesg='Missing address/url') from None
+        if proto != 'file' and host is None:
+            raise s_exc.BadTypeValu(valu=orig, name=self.name, mesg='Missing address/url')
+
+        if not hostparts and not pathpart:
+            raise s_exc.BadTypeValu(valu=orig, name=self.name,
+                                    mesg='Missing address/url') from None
 
         base = f'{proto}://{hostparts}{pathpart}'
         subs['base'] = base
@@ -2305,6 +2317,7 @@ class InetModule(s_module.CoreModule):
                         ('name:en', ('inet:user', {}), {
                             'doc': 'The English version of the name associated with the (may be different from '
                                    'the account identifier, e.g., a display name).',
+                            'deprecated': True,
                         }),
                         ('aliases', ('array', {'type': 'inet:user', 'uniq': True, 'sorted': True}), {
                             'doc': 'An array of alternate names for the user.',
@@ -2322,7 +2335,8 @@ class InetModule(s_module.CoreModule):
                             'doc': 'The localized version of the real name of the account owner / registrant.'
                         }),
                         ('realname:en', ('ps:name', {}), {
-                            'doc': 'The English version of the real name of the account owner / registrant.'
+                            'doc': 'The English version of the real name of the account owner / registrant.',
+                            'deprecated': True,
                         }),
                         ('signup', ('time', {}), {
                             'doc': 'The date and time the account was registered.'
@@ -2540,7 +2554,8 @@ class InetModule(s_module.CoreModule):
                         }),
                         ('name:en', ('inet:group', {}), {
                             'doc': 'The English version of the name associated with the group (may be different '
-                                   'from the localized name).'
+                                   'from the localized name).',
+                            'deprecated': True,
                         }),
                         ('url', ('inet:url', {}), {
                             'doc': 'The service provider URL where the group is hosted.'

synapse/models/orgs.py

@@ -202,6 +202,8 @@ class OuModule(s_module.CoreModule):
                 ('ou:jobtitle', ('str', {'lower': True, 'onespace': True}), {
                     'doc': 'A title for a position within an org.',
                 }),
+                ('ou:requirement', ('guid', {}), {
+                    'doc': 'A specific requirement.'}),
             ),
             'edges': (
                 (('ou:campaign', 'uses', 'ou:technique'), {
@@ -219,6 +221,8 @@ class OuModule(s_module.CoreModule):
                     'doc': 'The campaign made use of the target node.'}),
                 (('ou:contribution', 'includes', None), {
                     'doc': 'The contribution includes the specific node.'}),
+                ((None, 'meets', 'ou:requirement'), {
+                    'doc': 'The requirement is met by the source node.'}),
             ),
             'forms': (
                 ('ou:jobtype', {}, ()),
@@ -634,8 +638,10 @@ class OuModule(s_module.CoreModule):
                         'doc': 'Deprecated for scalability. Please use -(uses)> ou:technique.',
                     }),
                     ('goals', ('array', {'type': 'ou:goal', 'sorted': True, 'uniq': True}), {
-                        'doc': 'The assessed goals of the organization.'
-                    }),
+                        'doc': 'The assessed goals of the organization.'}),
+
+                    ('tag', ('syn:tag', {}), {
+                        'doc': 'A base tag used to encode assessments made by the organization.'}),
                 )),
                 ('ou:team', {}, (
                     ('org', ('ou:org', {}), {}),
@@ -708,6 +714,7 @@ class OuModule(s_module.CoreModule):
                         'doc': 'An array of alternative names for the industry.'}),
 
                     ('subs', ('array', {'type': 'ou:industry', 'split': ',', 'uniq': True, 'sorted': True}), {
+                        'deprecated': True,
                         'doc': 'Deprecated. Please use ou:industry:type taxonomy.'}),
 
                     ('sic', ('array', {'type': 'ou:sic', 'split': ',', 'uniq': True, 'sorted': True}), {
@@ -1122,6 +1129,45 @@ class OuModule(s_module.CoreModule):
                     }),
                     # TODO duration ('duration'
                 )),
+                ('ou:requirement', {}, (
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'A name for the requirement.'}),
+
+                    ('text', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'The text of the stated requirement.'}),
+
+                    ('optional', ('bool', {}), {
+                        'doc': 'Set to true if the requirement is optional.'}),
+
+                    ('priority', ('meta:priority', {}), {
+                        'doc': 'The priority of the requirement.'}),
+
+                    ('goal', ('ou:goal', {}), {
+                        'doc': 'The goal that the requirement is designed to achieve.'}),
+
+                    ('active', ('bool', {}), {
+                        'doc': 'Set to true if the requirement is currently active.'}),
+
+                    ('issued', ('time', {}), {
+                        'doc': 'The time that the requirement was first issued.'}),
+
+                    ('period', ('ival', {}), {
+                        'doc': 'The time window where the goal must be met. Can be ongoing.'}),
+
+                    ('issuer', ('ps:contact', {}), {
+                        'doc': 'The contact information of the entity which issued the requirement.'}),
+
+                    ('assignee', ('ps:contact', {}), {
+                        'doc': 'The contact information of the entity which is assigned to meet the requirement.'}),
+
+                    ('deps', ('array', {'type': 'ou:requirement', 'sorted': True, 'uniq': True}), {
+                        'doc': 'A list of sub-requirements which must be met to complete the requirement.'}),
+
+                    ('deps:min', ('int', {'min': 0}), {
+                        'doc': 'The minimum number dependant requirements which must be met. If unset, assume all must be met.'}),
+                )),
             )
         }