synapse 2.189.0__py311-none-any.whl → 2.191.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/common.py +9 -0
- synapse/cortex.py +26 -5
- synapse/datamodel.py +8 -1
- synapse/lib/cell.py +7 -7
- synapse/lib/msgpack.py +10 -3
- synapse/lib/storm.py +5 -1
- synapse/lib/stormhttp.py +3 -10
- synapse/lib/stormlib/model.py +37 -0
- synapse/lib/stormtypes.py +27 -17
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +2 -5
- synapse/models/auth.py +2 -1
- synapse/models/base.py +20 -0
- synapse/models/crypto.py +5 -2
- synapse/models/economic.py +45 -11
- synapse/models/inet.py +78 -21
- synapse/models/person.py +11 -4
- synapse/models/risk.py +6 -0
- synapse/models/syn.py +22 -12
- synapse/models/telco.py +3 -1
- synapse/tests/test_cortex.py +73 -31
- synapse/tests/test_lib_agenda.py +1 -6
- synapse/tests/test_lib_cell.py +28 -4
- synapse/tests/test_lib_httpapi.py +2 -4
- synapse/tests/test_lib_lmdbslab.py +1 -4
- synapse/tests/test_lib_stormhttp.py +9 -0
- synapse/tests/test_lib_stormlib_cortex.py +1 -3
- synapse/tests/test_lib_stormlib_log.py +1 -6
- synapse/tests/test_lib_stormlib_model.py +28 -0
- synapse/tests/test_lib_stormtypes.py +17 -3
- synapse/tests/test_lib_trigger.py +2 -3
- synapse/tests/test_model_base.py +12 -2
- synapse/tests/test_model_inet.py +23 -0
- synapse/tests/test_model_person.py +2 -0
- synapse/tests/test_model_risk.py +5 -0
- synapse/tests/test_model_syn.py +198 -0
- synapse/tests/test_utils.py +23 -4
- synapse/tests/utils.py +39 -5
- {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/METADATA +5 -5
- {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/RECORD +43 -43
- {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/LICENSE +0 -0
- {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/WHEEL +0 -0
- {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/top_level.txt +0 -0
synapse/common.py
CHANGED
|
@@ -1368,3 +1368,12 @@ def _timeout(delay):
|
|
|
1368
1368
|
"""
|
|
1369
1369
|
loop = asyncio.get_running_loop()
|
|
1370
1370
|
return _Timeout(loop.time() + delay if delay is not None else None)
|
|
1371
|
+
|
|
1372
|
+
def format(text, **kwargs):
|
|
1373
|
+
'''
|
|
1374
|
+
Similar to python str.format() but treats tokens as opaque.
|
|
1375
|
+
'''
|
|
1376
|
+
for name, valu in kwargs.items():
|
|
1377
|
+
tokn = '{' + name + '}'
|
|
1378
|
+
text = text.replace(tokn, valu)
|
|
1379
|
+
return text
|
synapse/cortex.py
CHANGED
|
@@ -4651,6 +4651,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4651
4651
|
# allow an admin to directly open the cortex hive
|
|
4652
4652
|
# (perhaps this should be a Cell() level pattern)
|
|
4653
4653
|
if path[0] == 'hive' and user.isAdmin():
|
|
4654
|
+
s_common.deprecated('Cortex /hive telepath path', curv='2.167.0')
|
|
4654
4655
|
return await self.hiveapi.anit(self.hive, user)
|
|
4655
4656
|
|
|
4656
4657
|
if path[0] == 'layer':
|
|
@@ -5830,14 +5831,18 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5830
5831
|
opts = self._initStormOpts(opts)
|
|
5831
5832
|
|
|
5832
5833
|
if self.stormpool is not None and opts.get('mirror', True):
|
|
5833
|
-
extra = await self.getLogExtra(text=text)
|
|
5834
5834
|
proxy = await self._getMirrorProxy(opts)
|
|
5835
5835
|
|
|
5836
5836
|
if proxy is not None:
|
|
5837
|
-
|
|
5837
|
+
proxname = proxy._ahainfo.get('name')
|
|
5838
|
+
extra = await self.getLogExtra(mirror=proxname, hash=s_storm.queryhash(text))
|
|
5839
|
+
logger.info(f'Offloading Storm query to mirror {proxname}.', extra=extra)
|
|
5838
5840
|
|
|
5839
5841
|
mirropts = await self._getMirrorOpts(opts)
|
|
5840
5842
|
|
|
5843
|
+
mirropts.setdefault('_loginfo', {})
|
|
5844
|
+
mirropts['_loginfo']['pool:from'] = self.ahasvcname
|
|
5845
|
+
|
|
5841
5846
|
try:
|
|
5842
5847
|
return await proxy.count(text, opts=mirropts)
|
|
5843
5848
|
|
|
@@ -5911,10 +5916,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5911
5916
|
proxy = await self._getMirrorProxy(opts)
|
|
5912
5917
|
|
|
5913
5918
|
if proxy is not None:
|
|
5914
|
-
|
|
5919
|
+
proxname = proxy._ahainfo.get('name')
|
|
5920
|
+
extra = await self.getLogExtra(mirror=proxname, hash=s_storm.queryhash(text))
|
|
5921
|
+
logger.info(f'Offloading Storm query to mirror {proxname}.', extra=extra)
|
|
5915
5922
|
|
|
5916
5923
|
mirropts = await self._getMirrorOpts(opts)
|
|
5917
5924
|
|
|
5925
|
+
mirropts.setdefault('_loginfo', {})
|
|
5926
|
+
mirropts['_loginfo']['pool:from'] = self.ahasvcname
|
|
5927
|
+
|
|
5918
5928
|
try:
|
|
5919
5929
|
async for mesg in proxy.storm(text, opts=mirropts):
|
|
5920
5930
|
yield mesg
|
|
@@ -5941,10 +5951,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5941
5951
|
proxy = await self._getMirrorProxy(opts)
|
|
5942
5952
|
|
|
5943
5953
|
if proxy is not None:
|
|
5944
|
-
|
|
5954
|
+
proxname = proxy._ahainfo.get('name')
|
|
5955
|
+
extra = await self.getLogExtra(mirror=proxname, hash=s_storm.queryhash(text))
|
|
5956
|
+
logger.info(f'Offloading Storm query to mirror {proxname}.', extra=extra)
|
|
5945
5957
|
|
|
5946
5958
|
mirropts = await self._getMirrorOpts(opts)
|
|
5947
5959
|
|
|
5960
|
+
mirropts.setdefault('_loginfo', {})
|
|
5961
|
+
mirropts['_loginfo']['pool:from'] = self.ahasvcname
|
|
5962
|
+
|
|
5948
5963
|
try:
|
|
5949
5964
|
return await proxy.callStorm(text, opts=mirropts)
|
|
5950
5965
|
except s_exc.TimeOut:
|
|
@@ -5966,10 +5981,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5966
5981
|
proxy = await self._getMirrorProxy(opts)
|
|
5967
5982
|
|
|
5968
5983
|
if proxy is not None:
|
|
5969
|
-
|
|
5984
|
+
proxname = proxy._ahainfo.get('name')
|
|
5985
|
+
extra = await self.getLogExtra(mirror=proxname, hash=s_storm.queryhash(text))
|
|
5986
|
+
logger.info(f'Offloading Storm query to mirror {proxname}.', extra=extra)
|
|
5970
5987
|
|
|
5971
5988
|
mirropts = await self._getMirrorOpts(opts)
|
|
5972
5989
|
|
|
5990
|
+
mirropts.setdefault('_loginfo', {})
|
|
5991
|
+
mirropts['_loginfo']['pool:from'] = self.ahasvcname
|
|
5992
|
+
|
|
5973
5993
|
try:
|
|
5974
5994
|
async for mesg in proxy.exportStorm(text, opts=mirropts):
|
|
5975
5995
|
yield mesg
|
|
@@ -6152,6 +6172,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
6152
6172
|
info['text'] = text
|
|
6153
6173
|
info['username'] = user.name
|
|
6154
6174
|
info['user'] = user.iden
|
|
6175
|
+
info['hash'] = s_storm.queryhash(text)
|
|
6155
6176
|
stormlogger.log(self.stormloglvl, 'Executing storm query {%s} as [%s]', text, user.name,
|
|
6156
6177
|
extra={'synapse': info})
|
|
6157
6178
|
|
synapse/datamodel.py
CHANGED
|
@@ -1114,7 +1114,14 @@ class Model:
|
|
|
1114
1114
|
if item == '$self':
|
|
1115
1115
|
return form.name
|
|
1116
1116
|
|
|
1117
|
-
|
|
1117
|
+
item = s_common.format(item, **template)
|
|
1118
|
+
|
|
1119
|
+
# warn but do not blow up. there may be extended model elements
|
|
1120
|
+
# with {}s which are not used for templates...
|
|
1121
|
+
if item.find('{') != -1: # pragma: no cover
|
|
1122
|
+
logger.warning(f'Missing template specifier in: {item}')
|
|
1123
|
+
|
|
1124
|
+
return item
|
|
1118
1125
|
|
|
1119
1126
|
if isinstance(item, dict):
|
|
1120
1127
|
return {convert(k): convert(v) for (k, v) in item.items()}
|
synapse/lib/cell.py
CHANGED
|
@@ -96,16 +96,16 @@ def adminapi(log=False):
|
|
|
96
96
|
def decrfunc(func):
|
|
97
97
|
|
|
98
98
|
@functools.wraps(func)
|
|
99
|
-
def wrapped(*args, **kwargs):
|
|
99
|
+
def wrapped(self, *args, **kwargs):
|
|
100
100
|
|
|
101
|
-
if
|
|
102
|
-
raise s_exc.AuthDeny(mesg='User is not an admin.',
|
|
103
|
-
user=
|
|
101
|
+
if self.user is not None and not self.user.isAdmin():
|
|
102
|
+
raise s_exc.AuthDeny(mesg=f'User is not an admin [{self.user.name}]',
|
|
103
|
+
user=self.user.iden, username=self.user.name)
|
|
104
104
|
if log:
|
|
105
|
-
logger.info('Executing [
|
|
106
|
-
func.__qualname__
|
|
105
|
+
logger.info(f'Executing [{func.__qualname__}] as [{self.user.name}] with args [{args}[{kwargs}]',
|
|
106
|
+
extra={'synapse': {'wrapped_func': func.__qualname__}})
|
|
107
107
|
|
|
108
|
-
return func(*args, **kwargs)
|
|
108
|
+
return func(self, *args, **kwargs)
|
|
109
109
|
|
|
110
110
|
wrapped.__syn_wrapped__ = 'adminapi'
|
|
111
111
|
|
synapse/lib/msgpack.py
CHANGED
|
@@ -26,8 +26,16 @@ def _ext_en(item):
|
|
|
26
26
|
return msgpack.ExtType(1, item.to_bytes(size, 'big', signed=True))
|
|
27
27
|
return item
|
|
28
28
|
|
|
29
|
+
_packer_kwargs = {
|
|
30
|
+
'use_bin_type': True,
|
|
31
|
+
'unicode_errors': 'surrogatepass',
|
|
32
|
+
'default': _ext_en,
|
|
33
|
+
}
|
|
34
|
+
if msgpack.version >= (1, 1, 0):
|
|
35
|
+
_packer_kwargs['buf_size'] = 1024 * 1024
|
|
36
|
+
|
|
29
37
|
# Single Packer object which is reused for performance
|
|
30
|
-
pakr = msgpack.Packer(
|
|
38
|
+
pakr = msgpack.Packer(**_packer_kwargs)
|
|
31
39
|
if isinstance(pakr, m_fallback.Packer): # pragma: no cover
|
|
32
40
|
logger.warning('******************************************************************************************************')
|
|
33
41
|
logger.warning('* msgpack is using the pure python fallback implementation. This will impact performance negatively. *')
|
|
@@ -87,8 +95,7 @@ def _fallback_en(item):
|
|
|
87
95
|
bytes: The serialized bytes in msgpack format.
|
|
88
96
|
'''
|
|
89
97
|
try:
|
|
90
|
-
return msgpack.packb(item,
|
|
91
|
-
unicode_errors='surrogatepass', default=_ext_en)
|
|
98
|
+
return msgpack.packb(item, **_packer_kwargs)
|
|
92
99
|
except TypeError as e:
|
|
93
100
|
mesg = f'{e.args[0]}: {repr(item)[:20]}'
|
|
94
101
|
raise s_exc.NotMsgpackSafe(mesg=mesg) from e
|
synapse/lib/storm.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import types
|
|
2
2
|
import pprint
|
|
3
3
|
import asyncio
|
|
4
|
+
import hashlib
|
|
4
5
|
import logging
|
|
5
6
|
import argparse
|
|
6
7
|
import contextlib
|
|
@@ -18,7 +19,6 @@ import synapse.lib.chop as s_chop
|
|
|
18
19
|
import synapse.lib.coro as s_coro
|
|
19
20
|
import synapse.lib.node as s_node
|
|
20
21
|
import synapse.lib.snap as s_snap
|
|
21
|
-
import synapse.lib.time as s_time
|
|
22
22
|
import synapse.lib.cache as s_cache
|
|
23
23
|
import synapse.lib.layer as s_layer
|
|
24
24
|
import synapse.lib.scope as s_scope
|
|
@@ -1681,6 +1681,10 @@ stormcmds = (
|
|
|
1681
1681
|
},
|
|
1682
1682
|
)
|
|
1683
1683
|
|
|
1684
|
+
@s_cache.memoize(size=1024)
|
|
1685
|
+
def queryhash(text):
|
|
1686
|
+
return hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
|
|
1687
|
+
|
|
1684
1688
|
class DmonManager(s_base.Base):
|
|
1685
1689
|
'''
|
|
1686
1690
|
Manager for StormDmon objects.
|
synapse/lib/stormhttp.py
CHANGED
|
@@ -292,13 +292,6 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
292
292
|
'codereason': self.codereason,
|
|
293
293
|
}
|
|
294
294
|
|
|
295
|
-
def strify(self, item):
|
|
296
|
-
if isinstance(item, (list, tuple)):
|
|
297
|
-
return [(str(k), str(v)) for (k, v) in item]
|
|
298
|
-
elif isinstance(item, dict):
|
|
299
|
-
return {str(k): str(v) for k, v in item.items()}
|
|
300
|
-
return item
|
|
301
|
-
|
|
302
295
|
@s_stormtypes.stormfunc(readonly=True)
|
|
303
296
|
async def urlencode(self, text):
|
|
304
297
|
text = await s_stormtypes.tostr(text)
|
|
@@ -341,7 +334,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
341
334
|
ssl_verify = await s_stormtypes.tobool(ssl_verify, noneok=True)
|
|
342
335
|
ssl_opts = await s_stormtypes.toprim(ssl_opts)
|
|
343
336
|
|
|
344
|
-
headers =
|
|
337
|
+
headers = s_stormtypes.strifyHttpArg(headers)
|
|
345
338
|
|
|
346
339
|
sock = await WebSocket.anit()
|
|
347
340
|
|
|
@@ -410,9 +403,9 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
410
403
|
|
|
411
404
|
kwargs = {'allow_redirects': allow_redirects}
|
|
412
405
|
if params:
|
|
413
|
-
kwargs['params'] =
|
|
406
|
+
kwargs['params'] = s_stormtypes.strifyHttpArg(params, multi=True)
|
|
414
407
|
|
|
415
|
-
headers =
|
|
408
|
+
headers = s_stormtypes.strifyHttpArg(headers)
|
|
416
409
|
|
|
417
410
|
if proxy is not None:
|
|
418
411
|
self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
|
synapse/lib/stormlib/model.py
CHANGED
|
@@ -916,6 +916,21 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
916
916
|
'desc': 'Do not copy nodedata to the inet:tls:servercert node.'},
|
|
917
917
|
),
|
|
918
918
|
'returns': {'type': 'node', 'desc': 'The newly created inet:tls:servercert node.'}}},
|
|
919
|
+
{'name': 'inetServiceMessageClientAddress', 'desc': '''
|
|
920
|
+
Migrate the :client:address property to :client on inet:service:message nodes.
|
|
921
|
+
|
|
922
|
+
Edits will be made to the inet:service:message node in the current write layer.
|
|
923
|
+
|
|
924
|
+
If the :client:address property is set and the :client property is not set,
|
|
925
|
+
the :client property will be set with the :client:address value. If both
|
|
926
|
+
properties are set, the value will be moved into nodedata under the key
|
|
927
|
+
'migration:inet:service:message:address'.
|
|
928
|
+
''',
|
|
929
|
+
'type': {'type': 'function', '_funcname': '_storm_query',
|
|
930
|
+
'args': (
|
|
931
|
+
{'name': 'n', 'type': 'node', 'desc': 'The inet:sevice:message node to migrate.'},
|
|
932
|
+
),
|
|
933
|
+
'returns': {'type': 'null'}}},
|
|
919
934
|
|
|
920
935
|
)
|
|
921
936
|
_storm_lib_path = ('model', 'migration', 's')
|
|
@@ -956,6 +971,28 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
956
971
|
|
|
957
972
|
return($node)
|
|
958
973
|
}
|
|
974
|
+
|
|
975
|
+
function inetServiceMessageClientAddress(n) {
|
|
976
|
+
$form = $n.form()
|
|
977
|
+
if ($form != 'inet:service:message') {
|
|
978
|
+
$mesg = `$lib.model.migration.s.inetServiceMessageClientAddress() only accepts inet:service:message nodes, not {$form}`
|
|
979
|
+
$lib.raise(BadArg, $mesg)
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
if (not $n.props.'client:address') { return() }
|
|
983
|
+
|
|
984
|
+
yield $n
|
|
985
|
+
|
|
986
|
+
if :client {
|
|
987
|
+
$node.data.set(migration:inet:service:message:client:address, :client:address)
|
|
988
|
+
} else {
|
|
989
|
+
[ :client = :client:address ]
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
[ -:client:address ]
|
|
993
|
+
|
|
994
|
+
return()
|
|
995
|
+
}
|
|
959
996
|
'''
|
|
960
997
|
|
|
961
998
|
def getObjLocals(self):
|
synapse/lib/stormtypes.py
CHANGED
|
@@ -60,6 +60,19 @@ def confirmEasyPerm(item, perm, mesg=None):
|
|
|
60
60
|
def allowedEasyPerm(item, perm):
|
|
61
61
|
return s_scope.get('runt').allowedEasyPerm(item, perm)
|
|
62
62
|
|
|
63
|
+
def strifyHttpArg(item, multi=False):
|
|
64
|
+
if isinstance(item, (list, tuple)):
|
|
65
|
+
return [(str(k), str(v)) for (k, v) in item]
|
|
66
|
+
elif isinstance(item, dict):
|
|
67
|
+
retn = {}
|
|
68
|
+
for name, valu in item.items():
|
|
69
|
+
if isinstance(valu, (list, tuple)) and multi:
|
|
70
|
+
retn[str(name)] = [str(v) for v in valu]
|
|
71
|
+
else:
|
|
72
|
+
retn[str(name)] = str(valu)
|
|
73
|
+
return retn
|
|
74
|
+
return item
|
|
75
|
+
|
|
63
76
|
class StormTypesRegistry:
|
|
64
77
|
# The following types are currently undefined.
|
|
65
78
|
base_undefined_types = (
|
|
@@ -2306,13 +2319,6 @@ class LibAxon(Lib):
|
|
|
2306
2319
|
'hashset': self.hashset,
|
|
2307
2320
|
}
|
|
2308
2321
|
|
|
2309
|
-
def strify(self, item):
|
|
2310
|
-
if isinstance(item, (list, tuple)):
|
|
2311
|
-
return [(str(k), str(v)) for (k, v) in item]
|
|
2312
|
-
elif isinstance(item, dict):
|
|
2313
|
-
return {str(k): str(v) for k, v in item.items()}
|
|
2314
|
-
return item
|
|
2315
|
-
|
|
2316
2322
|
@stormfunc(readonly=True)
|
|
2317
2323
|
async def readlines(self, sha256, errors='ignore'):
|
|
2318
2324
|
if not self.runt.allowed(('axon', 'get')):
|
|
@@ -2384,8 +2390,8 @@ class LibAxon(Lib):
|
|
|
2384
2390
|
if proxy is not None:
|
|
2385
2391
|
self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
|
|
2386
2392
|
|
|
2387
|
-
params =
|
|
2388
|
-
headers =
|
|
2393
|
+
params = strifyHttpArg(params, multi=True)
|
|
2394
|
+
headers = strifyHttpArg(headers)
|
|
2389
2395
|
|
|
2390
2396
|
await self.runt.snap.core.getAxon()
|
|
2391
2397
|
|
|
@@ -2423,8 +2429,8 @@ class LibAxon(Lib):
|
|
|
2423
2429
|
timeout = await toprim(timeout)
|
|
2424
2430
|
ssl_opts = await toprim(ssl_opts)
|
|
2425
2431
|
|
|
2426
|
-
params =
|
|
2427
|
-
headers =
|
|
2432
|
+
params = strifyHttpArg(params, multi=True)
|
|
2433
|
+
headers = strifyHttpArg(headers)
|
|
2428
2434
|
|
|
2429
2435
|
if proxy is not None:
|
|
2430
2436
|
self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy'))
|
|
@@ -5008,9 +5014,12 @@ class List(Prim):
|
|
|
5008
5014
|
{'name': 'valu', 'type': 'any', 'desc': 'The value to check.', },
|
|
5009
5015
|
),
|
|
5010
5016
|
'returns': {'type': 'boolean', 'desc': 'True if the item is in the list, false otherwise.', }}},
|
|
5011
|
-
{'name': 'pop', 'desc': 'Pop and return the
|
|
5017
|
+
{'name': 'pop', 'desc': 'Pop and return the entry at the specified index in the list. If no index is specified, pop the last entry.',
|
|
5012
5018
|
'type': {'type': 'function', '_funcname': '_methListPop',
|
|
5013
|
-
'
|
|
5019
|
+
'args': (
|
|
5020
|
+
{'name': 'index', 'type': 'int', 'desc': 'Index of entry to pop.', 'default': -1},
|
|
5021
|
+
),
|
|
5022
|
+
'returns': {'type': 'any', 'desc': 'The entry at the specified index in the list.', }}},
|
|
5014
5023
|
{'name': 'size', 'desc': 'Return the length of the list.',
|
|
5015
5024
|
'type': {'type': 'function', '_funcname': '_methListSize',
|
|
5016
5025
|
'returns': {'type': 'int', 'desc': 'The size of the list.', }}},
|
|
@@ -5153,11 +5162,12 @@ class List(Prim):
|
|
|
5153
5162
|
return prim in self.valu
|
|
5154
5163
|
|
|
5155
5164
|
@stormfunc(readonly=True)
|
|
5156
|
-
async def _methListPop(self):
|
|
5165
|
+
async def _methListPop(self, index=-1):
|
|
5166
|
+
index = await toint(index)
|
|
5157
5167
|
try:
|
|
5158
|
-
return self.valu.pop()
|
|
5159
|
-
except IndexError:
|
|
5160
|
-
mesg =
|
|
5168
|
+
return self.valu.pop(index)
|
|
5169
|
+
except IndexError as exc:
|
|
5170
|
+
mesg = str(exc)
|
|
5161
5171
|
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
5162
5172
|
|
|
5163
5173
|
@stormfunc(readonly=True)
|
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 191, 0)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = 'f6b07ada7cb68701e769e155735c9c004a404454'
|
synapse/lib/view.py
CHANGED
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
import shutil
|
|
2
2
|
import asyncio
|
|
3
|
-
import hashlib
|
|
4
3
|
import logging
|
|
5
|
-
import itertools
|
|
6
4
|
import collections
|
|
7
5
|
|
|
8
6
|
import synapse.exc as s_exc
|
|
9
7
|
import synapse.common as s_common
|
|
10
8
|
|
|
11
9
|
import synapse.lib.cell as s_cell
|
|
12
|
-
import synapse.lib.coro as s_coro
|
|
13
10
|
import synapse.lib.snap as s_snap
|
|
14
11
|
import synapse.lib.layer as s_layer
|
|
15
12
|
import synapse.lib.nexus as s_nexus
|
|
16
13
|
import synapse.lib.scope as s_scope
|
|
17
|
-
import synapse.lib.
|
|
14
|
+
import synapse.lib.storm as s_storm
|
|
18
15
|
import synapse.lib.scrape as s_scrape
|
|
19
16
|
import synapse.lib.msgpack as s_msgpack
|
|
20
17
|
import synapse.lib.schemas as s_schemas
|
|
@@ -1004,7 +1001,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1004
1001
|
if editformat not in ('nodeedits', 'count', 'none'):
|
|
1005
1002
|
raise s_exc.BadConfValu(mesg=f'invalid edit format, got {editformat}', name='editformat', valu=editformat)
|
|
1006
1003
|
|
|
1007
|
-
texthash =
|
|
1004
|
+
texthash = s_storm.queryhash(text)
|
|
1008
1005
|
|
|
1009
1006
|
async def runStorm():
|
|
1010
1007
|
cancelled = False
|
synapse/models/auth.py
CHANGED
|
@@ -43,7 +43,8 @@ class AuthModule(s_module.CoreModule):
|
|
|
43
43
|
'doc': 'The WiFi SSID that the credentials allow access to.',
|
|
44
44
|
}),
|
|
45
45
|
('web:acct', ('inet:web:acct', {}), {
|
|
46
|
-
'
|
|
46
|
+
'deprecated': True,
|
|
47
|
+
'doc': 'Deprecated. Use :service:account.',
|
|
47
48
|
}),
|
|
48
49
|
('service:account', ('inet:service:account', {}), {
|
|
49
50
|
'doc': 'The service account that the credentials allow access to.'}),
|
synapse/models/base.py
CHANGED
|
@@ -56,6 +56,10 @@ class BaseModule(s_module.CoreModule):
|
|
|
56
56
|
'interfaces': ('meta:taxonomy',),
|
|
57
57
|
'doc': 'A taxonomy of event types for meta:event nodes.'}),
|
|
58
58
|
|
|
59
|
+
('meta:ruleset:type:taxonomy', ('taxonomy', {}), {
|
|
60
|
+
'interfaces': ('meta:taxonomy',),
|
|
61
|
+
'doc': 'A taxonomy for meta:ruleset types.'}),
|
|
62
|
+
|
|
59
63
|
('meta:ruleset', ('guid', {}), {
|
|
60
64
|
'doc': 'A set of rules linked with -(has)> edges.'}),
|
|
61
65
|
|
|
@@ -99,6 +103,9 @@ class BaseModule(s_module.CoreModule):
|
|
|
99
103
|
'deprecated': True,
|
|
100
104
|
'doc': 'A generic digraph time edge to show relationships outside the model.'}),
|
|
101
105
|
|
|
106
|
+
('meta:activity', ('int', {'enums': prioenums, 'enums:strict': False}), {
|
|
107
|
+
'doc': 'A generic activity level enumeration.'}),
|
|
108
|
+
|
|
102
109
|
('meta:priority', ('int', {'enums': prioenums, 'enums:strict': False}), {
|
|
103
110
|
'doc': 'A generic priority enumeration.'}),
|
|
104
111
|
|
|
@@ -120,6 +127,9 @@ class BaseModule(s_module.CoreModule):
|
|
|
120
127
|
),
|
|
121
128
|
},
|
|
122
129
|
'doc': 'A node which represents an aggregate count of a specific type.'}),
|
|
130
|
+
|
|
131
|
+
('markdown', ('str', {}), {
|
|
132
|
+
'doc': 'A markdown string.'}),
|
|
123
133
|
),
|
|
124
134
|
'interfaces': (
|
|
125
135
|
('meta:taxonomy', {
|
|
@@ -182,6 +192,12 @@ class BaseModule(s_module.CoreModule):
|
|
|
182
192
|
|
|
183
193
|
('url', ('inet:url', {}), {
|
|
184
194
|
'doc': 'A URL which documents the meta source.'}),
|
|
195
|
+
|
|
196
|
+
('ingest:latest', ('time', {}), {
|
|
197
|
+
'doc': 'Used by ingest logic to capture the last time a feed ingest ran.'}),
|
|
198
|
+
|
|
199
|
+
('ingest:offset', ('int', {}), {
|
|
200
|
+
'doc': 'Used by ingest logic to capture the current ingest offset within a feed.'}),
|
|
185
201
|
)),
|
|
186
202
|
|
|
187
203
|
('meta:seen', {}, (
|
|
@@ -263,6 +279,10 @@ class BaseModule(s_module.CoreModule):
|
|
|
263
279
|
('meta:ruleset', {}, (
|
|
264
280
|
('name', ('str', {'lower': True, 'onespace': True}), {
|
|
265
281
|
'doc': 'A name for the ruleset.'}),
|
|
282
|
+
|
|
283
|
+
('type', ('meta:ruleset:type:taxonomy', {}), {
|
|
284
|
+
'doc': 'The ruleset type.'}),
|
|
285
|
+
|
|
266
286
|
('desc', ('str', {}), {
|
|
267
287
|
'disp': {'hint': 'text'},
|
|
268
288
|
'doc': 'A description of the ruleset.'}),
|
synapse/models/crypto.py
CHANGED
|
@@ -82,6 +82,11 @@ class CryptoModule(s_module.CoreModule):
|
|
|
82
82
|
'ex': 'btc',
|
|
83
83
|
}),
|
|
84
84
|
('crypto:currency:address', ('comp', {'fields': (('coin', 'crypto:currency:coin'), ('iden', 'str')), 'sepr': '/'}), {
|
|
85
|
+
|
|
86
|
+
'interfaces': ('econ:pay:instrument',),
|
|
87
|
+
'template': {
|
|
88
|
+
'instrument': 'crypto currency address'},
|
|
89
|
+
|
|
85
90
|
'doc': 'An individual crypto currency address.',
|
|
86
91
|
'ex': 'btc/1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2',
|
|
87
92
|
}),
|
|
@@ -364,8 +369,6 @@ class CryptoModule(s_module.CoreModule):
|
|
|
364
369
|
'doc': 'The coin specific address identifier.', 'ro': True, }),
|
|
365
370
|
('desc', ('str', {}), {
|
|
366
371
|
'doc': 'A free-form description of the address.'}),
|
|
367
|
-
('contact', ('ps:contact', {}), {
|
|
368
|
-
'doc': 'Contact information associated with the address.'}),
|
|
369
372
|
)),
|
|
370
373
|
|
|
371
374
|
('crypto:algorithm', {}, ()),
|
synapse/models/economic.py
CHANGED
|
@@ -23,6 +23,11 @@ class EconModule(s_module.CoreModule):
|
|
|
23
23
|
'doc': 'An Issuer Id Number (IIN).'}),
|
|
24
24
|
|
|
25
25
|
('econ:pay:card', ('guid', {}), {
|
|
26
|
+
|
|
27
|
+
'interfaces': ('econ:pay:instrument',),
|
|
28
|
+
'template': {
|
|
29
|
+
'instrument': 'payment card'},
|
|
30
|
+
|
|
26
31
|
'doc': 'A single payment card.'}),
|
|
27
32
|
|
|
28
33
|
('econ:purchase', ('guid', {}), {
|
|
@@ -71,6 +76,11 @@ class EconModule(s_module.CoreModule):
|
|
|
71
76
|
'doc': 'A bank account type taxonomy.'}),
|
|
72
77
|
|
|
73
78
|
('econ:bank:account', ('guid', {}), {
|
|
79
|
+
|
|
80
|
+
'interfaces': ('econ:pay:instrument',),
|
|
81
|
+
'template': {
|
|
82
|
+
'instrument': 'bank account'},
|
|
83
|
+
|
|
74
84
|
'doc': 'A bank account.'}),
|
|
75
85
|
|
|
76
86
|
('econ:bank:balance', ('guid', {}), {
|
|
@@ -87,6 +97,25 @@ class EconModule(s_module.CoreModule):
|
|
|
87
97
|
|
|
88
98
|
('econ:bank:swift:bic', ('str', {'regex': '[A-Z]{6}[A-Z0-9]{5}'}), {
|
|
89
99
|
'doc': 'A Society for Worldwide Interbank Financial Telecommunication (SWIFT) Business Identifier Code (BIC).'}),
|
|
100
|
+
|
|
101
|
+
('econ:pay:instrument', ('ndef', {'interface': 'econ:pay:instrument'}), {
|
|
102
|
+
'doc': 'A node which may act as a payment instrument.'}),
|
|
103
|
+
),
|
|
104
|
+
|
|
105
|
+
'interfaces': (
|
|
106
|
+
('econ:pay:instrument', {
|
|
107
|
+
|
|
108
|
+
'doc': 'An interface for forms which may act as a payment instrument.',
|
|
109
|
+
'template': {
|
|
110
|
+
'instrument': 'instrument',
|
|
111
|
+
},
|
|
112
|
+
|
|
113
|
+
'props': (
|
|
114
|
+
|
|
115
|
+
('contact', ('ps:contact', {}), {
|
|
116
|
+
'doc': 'The primary contact for the {instrument}.'}),
|
|
117
|
+
),
|
|
118
|
+
}),
|
|
90
119
|
),
|
|
91
120
|
|
|
92
121
|
'edges': (
|
|
@@ -134,9 +163,6 @@ class EconModule(s_module.CoreModule):
|
|
|
134
163
|
|
|
135
164
|
('account', ('econ:bank:account', {}), {
|
|
136
165
|
'doc': 'A bank account associated with the payment card.'}),
|
|
137
|
-
|
|
138
|
-
('contact', ('ps:contact', {}), {
|
|
139
|
-
'doc': 'The contact information associated with the payment card.'}),
|
|
140
166
|
)),
|
|
141
167
|
|
|
142
168
|
('econ:purchase', {}, (
|
|
@@ -209,17 +235,26 @@ class EconModule(s_module.CoreModule):
|
|
|
209
235
|
('from:cash', ('bool', {}), {
|
|
210
236
|
'doc': 'Set to true if the payment input was in cash.'}),
|
|
211
237
|
|
|
238
|
+
('to:instrument', ('econ:pay:instrument', {}), {
|
|
239
|
+
'doc': 'The payment instrument which received funds from the payment.'}),
|
|
240
|
+
|
|
241
|
+
('from:instrument', ('econ:pay:instrument', {}), {
|
|
242
|
+
'doc': 'The payment instrument used to make the payment.'}),
|
|
243
|
+
|
|
212
244
|
('from:account', ('econ:bank:account', {}), {
|
|
213
|
-
'
|
|
245
|
+
'deprecated': True,
|
|
246
|
+
'doc': 'Deprecated. Please use :from:instrument.'}),
|
|
214
247
|
|
|
215
248
|
('from:pay:card', ('econ:pay:card', {}), {
|
|
216
|
-
'
|
|
249
|
+
'deprecated': True,
|
|
250
|
+
'doc': 'Deprecated. Please use :from:instrument.'}),
|
|
217
251
|
|
|
218
252
|
('from:contract', ('ou:contract', {}), {
|
|
219
253
|
'doc': 'A contract used as an aggregate payment source.'}),
|
|
220
254
|
|
|
221
255
|
('from:coinaddr', ('crypto:currency:address', {}), {
|
|
222
|
-
'
|
|
256
|
+
'deprecated': True,
|
|
257
|
+
'doc': 'Deprecated. Please use :from:instrument.'}),
|
|
223
258
|
|
|
224
259
|
('from:contact', ('ps:contact', {}), {
|
|
225
260
|
'doc': 'Contact information for the entity making the payment.'}),
|
|
@@ -228,10 +263,12 @@ class EconModule(s_module.CoreModule):
|
|
|
228
263
|
'doc': 'Set to true if the payment output was in cash.'}),
|
|
229
264
|
|
|
230
265
|
('to:account', ('econ:bank:account', {}), {
|
|
231
|
-
'
|
|
266
|
+
'deprecated': True,
|
|
267
|
+
'doc': 'Deprecated. Please use :to:instrument.'}),
|
|
232
268
|
|
|
233
269
|
('to:coinaddr', ('crypto:currency:address', {}), {
|
|
234
|
-
'
|
|
270
|
+
'deprecated': True,
|
|
271
|
+
'doc': 'Deprecated. Please use :to:instrument.'}),
|
|
235
272
|
|
|
236
273
|
('to:contact', ('ps:contact', {}), {
|
|
237
274
|
'doc': 'Contact information for the person/org being paid.'}),
|
|
@@ -448,9 +485,6 @@ class EconModule(s_module.CoreModule):
|
|
|
448
485
|
('iban', ('econ:bank:iban', {}), {
|
|
449
486
|
'doc': 'The IBAN for the account.'}),
|
|
450
487
|
|
|
451
|
-
('contact', ('ps:contact', {}), {
|
|
452
|
-
'doc': 'The contact information associated with the bank account.'}),
|
|
453
|
-
|
|
454
488
|
('issuer', ('ou:org', {}), {
|
|
455
489
|
'doc': 'The bank which issued the account.'}),
|
|
456
490
|
|