synapse 2.205.0__py311-none-any.whl → 2.207.0__py311-none-any.whl

synapse/lib/modelrev.py

@@ -1455,7 +1455,7 @@ class ModelMigration_0_2_31:
 
         # Pick up and classify all bad CPE nodes
         for idx, layer in enumerate(self.layers):
-            logger.debug(f'Classifying nodes in layer {idx}')
+            logger.debug('Classifying nodes in layer %s %s', idx, layer.iden)
 
             async for buid, sode in layer.getStorNodesByForm('it:sec:cpe'):
 
@@ -1499,7 +1499,7 @@ class ModelMigration_0_2_31:
 
         # Pick up all related CPE node info. The majority of the work happens in this loop
         for idx, layer in enumerate(self.layers):
-            logger.debug(f'Processing nodes in layer {idx}')
+            logger.debug('Processing nodes in layer %s %s', idx, layer.iden)
 
             for buid, node in self.nodes.items():
                 await self._loadNode(layer, buid, node=node)
@@ -1541,7 +1541,7 @@ class ModelMigration_0_2_31:
             await self.todos.clear()
 
             for idx, layer in enumerate(self.layers):
-                logger.debug(f'Processing references in layer {idx}')
+                logger.debug('Processing references in layer %s %s', idx, layer.iden)
 
                 async for entry in todotmp:
                     match entry:

synapse/lib/schemas.py

@@ -1128,3 +1128,14 @@ _reqValidOauth2TokenResponseSchema = {
     'required': ['access_token', 'expires_in'],
 }
 reqValidOauth2TokenResponse = s_config.getJsValidator(_reqValidOauth2TokenResponseSchema)
+
+_httpLoginV1Schema = {
+    'type': 'object',
+    'properties': {
+        'user': {'type': 'string'},
+        'passwd': {'type': 'string'},
+        },
+    'additionalProperties': False,
+    'required': ['user', 'passwd'],
+}
+reqValidHttpLoginV1 = s_config.getJsValidator(_httpLoginV1Schema)

synapse/lib/spooled.py

@@ -8,6 +8,7 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.lmdbslab as s_lmdbslab
 
 MAX_SPOOL_SIZE = 10000
+DEFAULT_MAPSIZE = s_const.mebibyte * 32
 
 class Spooled(s_base.Base):
     '''
@@ -48,7 +49,7 @@ class Spooled(s_base.Base):
 
         slabpath = tempfile.mkdtemp(dir=dirn, prefix='spooled_', suffix='.lmdb')
 
-        self.slab = await s_lmdbslab.Slab.anit(slabpath, map_size=s_const.mebibyte * 32)
+        self.slab = await s_lmdbslab.Slab.anit(slabpath, map_size=DEFAULT_MAPSIZE)
         if self.cell is not None:
             self.slab.addResizeCallback(self.cell.checkFreeSpace)
 

synapse/lib/stormhttp.py

@@ -457,23 +457,23 @@ class LibHttp(s_stormtypes.Lib):
                         hnfo = {
                             'code': hist.status,
                             'reason': await self.codereason(hist.status),
-                            'headers': dict(hist.headers),
+                            'headers': {str(k): v for k, v in hist.headers.items()},
                             'url': str(hist.url),
                             # aiohttp has already closed the connection by this point
                             # so there is no connection to read a body from.
                             'body': b'',
                             'history': [],
-                            'request_headers': dict(hist.request_info.headers)
+                            'request_headers': {str(k): v for k, v in hist.request_info.headers.items()}
                         }
                         history.append(hnfo)
                     info = {
                         'code': resp.status,
                         'reason': await self.codereason(resp.status),
-                        'headers': dict(resp.headers),
+                        'headers': {str(k): v for k, v in resp.headers.items()},
                         'url': str(resp.url),
                         'body': await resp.read(),
                         'history': history,
-                        'request_headers': dict(resp.request_info.headers)
+                        'request_headers': {str(k): v for k, v in resp.request_info.headers.items()},
                     }
                     return HttpResp(info)
 
@@ -492,11 +492,11 @@ class LibHttp(s_stormtypes.Lib):
                     'err': err,
                     'code': -1,
                     'reason': reason,
-                    'headers': dict(),
+                    'headers': {},
                     'url': url,
                     'body': b'',
                     'history': [],
-                    'request_headers': dict(),
+                    'request_headers': {},
                 }
                 return HttpResp(info)
 

synapse/lib/stormlib/aha.py

@@ -691,6 +691,7 @@ The ready column indicates that a service has entered into the realtime change w
                     {"name": "host", "width": 16},
                     {"name": "port", "width": 8},
                     {"name": "version", "width": 12},
+                    {"name": "synapse", "width": 12},
                     {"name": "nexus idx", "width": 10},
                 ],
                 "separators": {
@@ -732,6 +733,7 @@ The ready column indicates that a service has entered into the realtime change w
                     'host': $svcinfo.urlinfo.host,
                     'port': $svcinfo.urlinfo.port,
                     'version': '<unknown>',
+                    'synapse_version': '<unknown>',
                     'nexs_indx': (0)
                 })
                 if ($cell_infos.$svcname) {
@@ -743,7 +745,8 @@ The ready column indicates that a service has entered into the realtime change w
                     } else {
                         $status.role = 'leader'
                     }
-                    $status.version = $info.synapse.verstring
+                    $status.version = $info.cell.verstring
+                    $status.synapse_version = $info.synapse.verstring
                 }
                 $group_status.append($status)
             }
@@ -779,6 +782,7 @@ The ready column indicates that a service has entered into the realtime change w
                     $status.host,
                     $status.port,
                     $status.version,
+                    $status.synapse_version,
                     $status.nexs_indx
                 )
                 $lib.print($printer.row($row))

synapse/lib/stormlib/model.py

@@ -1181,7 +1181,7 @@ class LibModelMigrations_0_2_31(s_stormtypes.Lib):
                 await self.runt.printf(f'    layer: {layriden}')
                 for iden, refinfo in reflist:
                     form, prop, *_ = refinfo
-                    await self.runt.printf(f'      - {form}:{prop} (iden: {iden}')
+                    await self.runt.printf(f'      - {form}:{prop} (iden: {iden})')
 
         n1edges = node['n1edges']
         n2edges = node['n2edges']

synapse/lib/stormtypes.py

@@ -1235,7 +1235,7 @@ class LibBase(Lib):
             Examples:
                 Fire an event called ``demo`` with some data::
 
-                    cli> storm $foo='bar' $lib.fire('demo', foo=$foo, knight='ni')
+                    storm> $foo='bar' $lib.fire('demo', foo=$foo, knight='ni')
                     ...
                     ('storm:fire', {'type': 'demo', 'data': {'foo': 'bar', 'knight': 'ni'}})
                     ...
@@ -1268,7 +1268,7 @@ class LibBase(Lib):
             Examples:
                 Create a dictionary object with a key whose value is null, and call ``$lib.fire()`` with it::
 
-                    cli> storm $d=({"key": $lib.null}) $lib.fire('demo', d=$d)
+                    storm> $d=({"key": $lib.null}) $lib.fire('demo', d=$d)
                     ('storm:fire', {'type': 'demo', 'data': {'d': {'key': None}}})
             ''',
             'type': 'null', },
@@ -1295,7 +1295,7 @@ class LibBase(Lib):
             Examples:
                 Conditionally print a statement based on the constant value::
 
-                    cli> storm if $lib.true { $lib.print('Is True') } else { $lib.print('Is False') }
+                    storm> if $lib.true { $lib.print('Is True') } else { $lib.print('Is False') }
                     Is True
                 ''',
          'type': 'boolean', },
@@ -1305,7 +1305,7 @@ class LibBase(Lib):
             Examples:
                 Conditionally print a statement based on the constant value::
 
-                    cli> storm if $lib.false { $lib.print('Is True') } else { $lib.print('Is False') }
+                    storm> if $lib.false { $lib.print('Is True') } else { $lib.print('Is False') }
                     Is False''',
          'type': 'boolean', },
         {'name': 'text', 'desc': 'Get a Storm Text object. This is deprecated; please use a list to append strings to, and then use ``$lib.str.join()`` to join them on demand.',
@@ -1344,19 +1344,19 @@ class LibBase(Lib):
             Examples:
                 Print a simple string::
 
-                    cli> storm $lib.print("Hello world!")
+                    storm> $lib.print("Hello world!")
                     Hello world!
 
                 Format and print string based on variables::
 
-                    cli> storm $d=({"key1": (1), "key2": "two"})
+                    storm> $d=({"key1": (1), "key2": "two"})
                          for ($key, $value) in $d { $lib.print('{k} => {v}', k=$key, v=$value) }
                     key1 => 1
                     key2 => two
 
                 Use values off of a node to format and print string::
 
-                    cli> storm inet:ipv4:asn
+                    storm> inet:ipv4:asn
                          $lib.print("node: {ndef}, asn: {asn}", ndef=$node.ndef(), asn=:asn) | spin
                     node: ('inet:ipv4', 16909060), asn: 1138
 
@@ -1376,7 +1376,7 @@ class LibBase(Lib):
         Examples:
             Generate a sequence of integers based on the size of an array::
 
-                cli> storm $a=(foo,bar,(2)) for $i in $lib.range($lib.len($a)) {$lib.fire('test', indx=$i, valu=$a.$i)}
+                storm> $a=(foo,bar,(2)) for $i in $lib.range($lib.len($a)) {$lib.fire('test', indx=$i, valu=$a.$i)}
                 Executing query at 2021/03/22 19:25:48.835
                 ('storm:fire', {'type': 'test', 'data': {'index': 0, 'valu': 'foo'}})
                 ('storm:fire', {'type': 'test', 'data': {'index': 1, 'valu': 'bar'}})
@@ -2029,7 +2029,7 @@ class LibStr(Lib):
             Examples:
                 Join together a list of strings with a dot separator::
 
-                    cli> storm $foo=$lib.str.join('.', ('rep', 'vtx', 'tag')) $lib.print($foo)
+                    storm> $foo=$lib.str.join('.', ('rep', 'vtx', 'tag')) $lib.print($foo)
 
                     rep.vtx.tag''',
          'type': {'type': 'function', '_funcname': 'join',
@@ -2050,7 +2050,7 @@ class LibStr(Lib):
             Examples:
                 Format a string with a fixed argument and a variable::
 
-                    cli> storm $list=(1,2,3,4)
+                    storm> $list=(1,2,3,4)
                          $str=$lib.str.format('Hello {name}, your list is {list}!', name='Reader', list=$list)
                          $lib.print($str)
 
@@ -2324,7 +2324,7 @@ class LibAxon(Lib):
             Examples:
                 Save a base64 encoded buffer to the Axon::
 
-                    cli> storm $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.axon.put($buf)
+                    storm> $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.axon.put($buf)
                          $lib.print('size={size} sha256={sha256}', size=$size, sha256=$sha256)
 
                     size=4 sha256=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08''',
@@ -2340,7 +2340,7 @@ class LibAxon(Lib):
                 Check if the Axon has a given file::
 
                     # This example assumes the Axon does have the bytes
-                    cli> storm if $lib.axon.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) {
+                    storm> if $lib.axon.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) {
                             $lib.print("Has bytes")
                         } else {
                             $lib.print("Does not have bytes")
@@ -2812,7 +2812,7 @@ class LibBytes(Lib):
             Examples:
                 Save a base64 encoded buffer to the Axon::
 
-                    cli> storm $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.bytes.put($buf)
+                    storm> $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.bytes.put($buf)
                          $lib.print('size={size} sha256={sha256}', size=$size, sha256=$sha256)
 
                     size=4 sha256=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08''',
@@ -2828,7 +2828,7 @@ class LibBytes(Lib):
                 Check if the Axon has a given file::
 
                     # This example assumes the Axon does have the bytes
-                    cli> storm if $lib.bytes.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) {
+                    storm> if $lib.bytes.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) {
                             $lib.print("Has bytes")
                         } else {
                             $lib.print("Does not have bytes")
@@ -3002,7 +3002,7 @@ class LibTime(Lib):
             Examples:
                 Convert a timestamp from seconds to millis and format it::
 
-                    cli> storm $seconds=1594684800 $millis=$lib.time.fromunix($seconds)
+                    storm> $seconds=1594684800 $millis=$lib.time.fromunix($seconds)
                          $str=$lib.time.format($millis, '%A %d, %B %Y') $lib.print($str)
 
                     Tuesday 14, July 2020''',
@@ -3017,7 +3017,7 @@ class LibTime(Lib):
             Examples:
                 Parse a string as for its month/day/year value into a timestamp::
 
-                    cli> storm $s='06/01/2020' $ts=$lib.time.parse($s, '%m/%d/%Y') $lib.print($ts)
+                    storm> $s='06/01/2020' $ts=$lib.time.parse($s, '%m/%d/%Y') $lib.print($ts)
 
                     1590969600000''',
          'type': {'type': 'function', '_funcname': '_parse',
@@ -3034,7 +3034,7 @@ class LibTime(Lib):
             Examples:
                 Format a timestamp into a string::
 
-                    cli> storm $now=$lib.time.now() $str=$lib.time.format($now, '%A %d, %B %Y') $lib.print($str)
+                    storm> $now=$lib.time.now() $str=$lib.time.format($now, '%A %d, %B %Y') $lib.print($str)
 
                     Tuesday 14, July 2020''',
          'type': {'type': 'function', '_funcname': '_format',
@@ -3933,6 +3933,16 @@ class LibQueue(Lib):
                   'returns': {'type': 'list',
                               'desc': 'A list of queue definitions the current user is allowed to interact with.', }}},
     )
+    _storm_lib_perms = (
+        {'perm': ('queue', 'add'), 'gate': 'cortex',
+         'desc': 'Permits a user to create a named queue.'},
+        {'perm': ('queue', 'get'), 'gate': 'queue',
+         'desc': 'Permits a user to access a queue. This allows the user to read from the queue and remove items from it.'},
+        {'perm': ('queue', 'put'), 'gate': 'queue',
+         'desc': 'Permits a user to put items into a queue.'},
+        {'perm': ('queue', 'del'), 'gate': 'queue',
+         'desc': 'Permits a user to delete a queue.'},
+    )
     _storm_lib_path = ('queue',)
 
     def getObjLocals(self):
@@ -4649,6 +4659,26 @@ class Str(Prim):
         {'name': 'json', 'desc': 'Parse a JSON string and return the deserialized data.',
          'type': {'type': 'function', '_funcname': '_methStrJson', 'args': (),
                   'returns': {'type': 'prim', 'desc': 'The JSON deserialized object.', }}},
+        {'name': 'join', 'desc': '''
+                Join items into a string using the current string as a separator.
+
+                Examples:
+                    Join together a list of strings with a dot separator::
+
+                        storm> $sepr='.' $foo=$sepr.join(('rep', 'vtx', 'tag')) $lib.print($foo)
+
+                        rep.vtx.tag
+
+                    Join values inline together with a dot separator::
+
+                        storm> $foo=('.').join(('rep', 'vtx', 'tag')) $lib.print($foo)
+
+                        rep.vtx.tag''',
+         'type': {'type': 'function', '_funcname': '_methStrJoin',
+                  'args': (
+                      {'name': 'items', 'type': 'list', 'desc': 'A list of items to join together.', },
+                  ),
+                  'returns': {'type': 'str', 'desc': 'The joined string.', }}},
     )
     _storm_typename = 'str'
     _ismutable = False
@@ -4679,6 +4709,7 @@ class Str(Prim):
             'reverse': self._methStrReverse,
             'format': self._methStrFormat,
             'json': self._methStrJson,
+            'join': self._methStrJoin,
         }
 
     def __int__(self):
@@ -4798,6 +4829,11 @@ class Str(Prim):
     async def _methStrJson(self):
         return s_json.loads(self.valu)
 
+    @stormfunc(readonly=True)
+    async def _methStrJoin(self, items):
+        strs = [await tostr(item) async for item in toiter(items)]
+        return self.valu.join(strs)
+
 @registry.registerType
 class Bytes(Prim):
     '''

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 205, 0)
+version = (2, 207, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'd985f8b93669529b2d90895baabb1dc289e287df'
+commit = 'b5307edb2fd8ba01d47f5f38bcd6033237deda7e'

synapse/models/base.py

@@ -188,6 +188,15 @@ class BaseModule(s_module.CoreModule):
                 (('meta:ruleset', 'has', 'meta:rule'), {
                     'doc': 'The meta:ruleset includes the meta:rule.'}),
 
+                (('meta:ruleset', 'has', 'inet:service:rule'), {
+                    'doc': 'The meta:ruleset includes the inet:service:rule.'}),
+
+                (('meta:ruleset', 'has', 'it:app:snort:rule'), {
+                    'doc': 'The meta:ruleset includes the it:app:snort:rule.'}),
+
+                (('meta:ruleset', 'has', 'it:app:yara:rule'), {
+                    'doc': 'The meta:ruleset includes the it:app:yara:rule.'}),
+
                 (('meta:rule', 'matches', None), {
                     'doc': 'The meta:rule has matched on target node.'}),
 

synapse/models/inet.py

@@ -108,12 +108,20 @@ class Addr(s_types.Str):
         s_types.Str.postTypeInit(self)
         self.setNormFunc(str, self._normPyStr)
 
+        self.defport = self.opts.get('defport', None)
+        self.defproto = self.opts.get('defproto', 'tcp')
+
     def _getPort(self, valu):
+
         parts = valu.split(':', 1)
         if len(parts) == 2:
             valu, port = parts
             port = self.modl.type('inet:port').norm(port)[0]
             return valu, port, f':{port}'
+
+        if self.defport:
+            return valu, self.defport, f':{self.defport}'
+
         return valu, None, ''
 
     def _normPyStr(self, valu):
@@ -123,7 +131,7 @@ class Addr(s_types.Str):
         # no protos use case sensitivity yet...
         valu = valu.lower()
 
-        proto = 'tcp'
+        proto = self.defproto
         parts = valu.split('://', 1)
         if len(parts) == 2:
             proto, valu = parts

synapse/models/infotech.py

@@ -1296,6 +1296,11 @@ class ItModule(s_module.CoreModule):
                     ('net6', ('inet:net6', {}), {
                         'doc': 'The optional contiguous IPv6 address range of this network.'}),
 
+                    ('dns:resolvers', ('array', {'type': 'inet:server',
+                                                 'typeopts': {'defport': 53, 'defproto': 'udp'},
+                                                 'sorted': True, 'uniq': True}), {
+                        'doc': 'An array of DNS servers configured to resolve requests for hosts on the network.'})
+
                 )),
                 ('it:account', {}, (
                     ('user', ('inet:user', {}), {

synapse/models/telco.py

@@ -193,6 +193,11 @@ class TelcoModule(s_module.CoreModule):
                     'doc': 'A mobile cell site which a phone may connect to.'
                 }),
 
+                # TODO - eventually break out ISO-3 country code into a sub
+                # https://en.wikipedia.org/wiki/TADIG_code
+                ('tel:mob:tadig', ('str', {'regex': '^[A-Z0-9]{5}$', 'strip': True}), {
+                    'doc': 'A Transferred Account Data Interchange Group number issued to a GSM carrier.'}),
+
             ),
 
             'forms': (
@@ -322,6 +327,9 @@ class TelcoModule(s_module.CoreModule):
                     ('loc', ('loc', {}), {
                         'doc': 'Location the carrier operates from.'
                     }),
+
+                    ('tadig', ('tel:mob:tadig', {}), {
+                        'doc': 'The TADIG code issued to the carrier.'}),
                 )),
                 ('tel:mob:cell', {}, (
                     ('carrier', ('tel:mob:carrier', {}), {'doc': 'Mobile carrier.', 'ro': True, }),
@@ -340,6 +348,8 @@ class TelcoModule(s_module.CoreModule):
                         'doc': 'The place associated with the latlong property.'}),
                 )),
 
+                ('tel:mob:tadig', {}, ()),
+
                 ('tel:mob:telem', {}, (
 
                     ('time', ('time', {}), {}),