synapse 2.174.0__py311-none-any.whl → 2.176.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +1 -0
- synapse/daemon.py +11 -12
- synapse/lib/ast.py +10 -5
- synapse/lib/cell.py +22 -13
- synapse/lib/jupyter.py +21 -0
- synapse/lib/layer.py +124 -30
- synapse/lib/link.py +3 -2
- synapse/lib/modelrev.py +15 -1
- synapse/lib/snap.py +13 -0
- synapse/lib/storm.py +112 -90
- synapse/lib/stormlib/storm.py +117 -5
- synapse/lib/types.py +1 -1
- synapse/lib/version.py +2 -2
- synapse/models/inet.py +23 -0
- synapse/tests/test_daemon.py +36 -0
- synapse/tests/test_lib_ast.py +35 -0
- synapse/tests/test_lib_cell.py +11 -0
- synapse/tests/test_lib_modelrev.py +29 -0
- synapse/tests/test_lib_storm.py +127 -11
- synapse/tests/test_lib_stormlib_storm.py +82 -1
- synapse/tests/test_model_inet.py +31 -0
- synapse/tools/changelog.py +27 -7
- {synapse-2.174.0.dist-info → synapse-2.176.0.dist-info}/METADATA +1 -1
- {synapse-2.174.0.dist-info → synapse-2.176.0.dist-info}/RECORD +27 -27
- {synapse-2.174.0.dist-info → synapse-2.176.0.dist-info}/WHEEL +1 -1
- {synapse-2.174.0.dist-info → synapse-2.176.0.dist-info}/LICENSE +0 -0
- {synapse-2.174.0.dist-info → synapse-2.176.0.dist-info}/top_level.txt +0 -0
synapse/lib/storm.py
CHANGED
|
@@ -14,6 +14,7 @@ import synapse.datamodel as s_datamodel
|
|
|
14
14
|
import synapse.lib.ast as s_ast
|
|
15
15
|
import synapse.lib.base as s_base
|
|
16
16
|
import synapse.lib.chop as s_chop
|
|
17
|
+
import synapse.lib.coro as s_coro
|
|
17
18
|
import synapse.lib.node as s_node
|
|
18
19
|
import synapse.lib.snap as s_snap
|
|
19
20
|
import synapse.lib.time as s_time
|
|
@@ -1873,6 +1874,14 @@ class Runtime(s_base.Base):
|
|
|
1873
1874
|
valu.incref()
|
|
1874
1875
|
self.vars.update(varz)
|
|
1875
1876
|
|
|
1877
|
+
self._initRuntVars(query)
|
|
1878
|
+
|
|
1879
|
+
self.proxies = {}
|
|
1880
|
+
|
|
1881
|
+
self.onfini(self._onRuntFini)
|
|
1882
|
+
|
|
1883
|
+
def _initRuntVars(self, query):
|
|
1884
|
+
|
|
1876
1885
|
# declare path builtins as non-runtsafe
|
|
1877
1886
|
self.runtvars = {
|
|
1878
1887
|
'node': False,
|
|
@@ -1881,17 +1890,14 @@ class Runtime(s_base.Base):
|
|
|
1881
1890
|
|
|
1882
1891
|
# inherit runtsafe vars from our root
|
|
1883
1892
|
if self.root is not None:
|
|
1884
|
-
self.runtvars.update(root.runtvars)
|
|
1893
|
+
self.runtvars.update(self.root.runtvars)
|
|
1885
1894
|
self.runtvars.update({k: True for k in self.root.getScopeVars().keys()})
|
|
1886
1895
|
|
|
1887
1896
|
# all vars/ctors are de-facto runtsafe
|
|
1888
1897
|
self.runtvars.update({k: True for k in self.vars.keys()})
|
|
1889
1898
|
self.runtvars.update({k: True for k in self.ctors.keys()})
|
|
1890
1899
|
|
|
1891
|
-
self.proxies = {}
|
|
1892
|
-
|
|
1893
1900
|
self._loadRuntVars(query)
|
|
1894
|
-
self.onfini(self._onRuntFini)
|
|
1895
1901
|
|
|
1896
1902
|
def getScopeVars(self):
|
|
1897
1903
|
'''
|
|
@@ -3491,13 +3497,18 @@ class DiffCmd(Cmd):
|
|
|
3491
3497
|
// Lift the nodes with the tag #cno.mal.redtree added in the top layer.
|
|
3492
3498
|
|
|
3493
3499
|
diff --tag cno.mal.redtree
|
|
3500
|
+
|
|
3501
|
+
// Lift nodes by multiple tags (results are uniqued)
|
|
3502
|
+
|
|
3503
|
+
diff --tag cno.mal.redtree rep.vt
|
|
3494
3504
|
'''
|
|
3495
3505
|
name = 'diff'
|
|
3496
3506
|
readonly = True
|
|
3497
3507
|
|
|
3498
3508
|
def getArgParser(self):
|
|
3499
3509
|
pars = Cmd.getArgParser(self)
|
|
3500
|
-
pars.add_argument('--tag', default=None,
|
|
3510
|
+
pars.add_argument('--tag', default=None, nargs='*',
|
|
3511
|
+
help='Lift only nodes with the given tag (or tags) in the top layer.')
|
|
3501
3512
|
pars.add_argument('--prop', default=None, help='Lift nodes with changes to the given property the top layer.')
|
|
3502
3513
|
return pars
|
|
3503
3514
|
|
|
@@ -3516,10 +3527,11 @@ class DiffCmd(Cmd):
|
|
|
3516
3527
|
|
|
3517
3528
|
if self.opts.tag:
|
|
3518
3529
|
|
|
3519
|
-
|
|
3530
|
+
tagnames = [await s_stormtypes.tostr(tag) for tag in self.opts.tag]
|
|
3520
3531
|
|
|
3521
3532
|
layr = runt.snap.view.layers[0]
|
|
3522
|
-
|
|
3533
|
+
|
|
3534
|
+
async for _, buid, sode in layr.liftByTags(tagnames):
|
|
3523
3535
|
node = await self.runt.snap._joinStorNode(buid, {layr.iden: sode})
|
|
3524
3536
|
if node is not None:
|
|
3525
3537
|
yield node, runt.initPath(node)
|
|
@@ -3765,73 +3777,76 @@ class MergeCmd(Cmd):
|
|
|
3765
3777
|
def _getPropFilter(self):
|
|
3766
3778
|
if self.opts.include_props:
|
|
3767
3779
|
|
|
3780
|
+
_include_props = set(self.opts.include_props)
|
|
3781
|
+
|
|
3768
3782
|
def propfilter(prop):
|
|
3769
|
-
|
|
3770
|
-
return False
|
|
3771
|
-
return True
|
|
3783
|
+
return prop not in _include_props
|
|
3772
3784
|
|
|
3773
3785
|
return propfilter
|
|
3774
3786
|
|
|
3775
3787
|
if self.opts.exclude_props:
|
|
3776
3788
|
|
|
3789
|
+
_exclude_props = set(self.opts.exclude_props)
|
|
3790
|
+
|
|
3777
3791
|
def propfilter(prop):
|
|
3778
|
-
|
|
3779
|
-
return True
|
|
3780
|
-
return False
|
|
3792
|
+
return prop in _exclude_props
|
|
3781
3793
|
|
|
3782
3794
|
return propfilter
|
|
3783
3795
|
|
|
3784
3796
|
return None
|
|
3785
3797
|
|
|
3786
|
-
async def _checkNodePerms(self, node, sode, runt):
|
|
3798
|
+
async def _checkNodePerms(self, node, sode, runt, allows):
|
|
3787
3799
|
|
|
3788
3800
|
layr0 = runt.snap.view.layers[0].iden
|
|
3789
3801
|
layr1 = runt.snap.view.layers[1].iden
|
|
3790
3802
|
|
|
3791
|
-
if sode.get('valu') is not None:
|
|
3803
|
+
if not allows['forms'] and sode.get('valu') is not None:
|
|
3792
3804
|
runt.confirm(('node', 'del', node.form.name), gateiden=layr0)
|
|
3793
3805
|
runt.confirm(('node', 'add', node.form.name), gateiden=layr1)
|
|
3794
3806
|
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3807
|
+
if not allows['props']:
|
|
3808
|
+
for name in sode.get('props', {}).keys():
|
|
3809
|
+
prop = node.form.prop(name)
|
|
3810
|
+
runt.confirmPropDel(prop, layriden=layr0)
|
|
3811
|
+
runt.confirmPropSet(prop, layriden=layr1)
|
|
3812
|
+
|
|
3813
|
+
if not allows['tags']:
|
|
3814
|
+
|
|
3815
|
+
tags = []
|
|
3816
|
+
tagadds = []
|
|
3817
|
+
for tag, valu in sode.get('tags', {}).items():
|
|
3818
|
+
if valu != (None, None):
|
|
3819
|
+
tagadds.append(tag)
|
|
3820
|
+
tagperm = tuple(tag.split('.'))
|
|
3821
|
+
runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
|
|
3822
|
+
runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
|
|
3823
|
+
else:
|
|
3824
|
+
tags.append((len(tag), tag))
|
|
3825
|
+
|
|
3826
|
+
for _, tag in sorted(tags, reverse=True):
|
|
3827
|
+
look = tag + '.'
|
|
3828
|
+
if any([tagadd.startswith(look) for tagadd in tagadds]):
|
|
3829
|
+
continue
|
|
3799
3830
|
|
|
3800
|
-
tags = []
|
|
3801
|
-
tagadds = []
|
|
3802
|
-
for tag, valu in sode.get('tags', {}).items():
|
|
3803
|
-
if valu != (None, None):
|
|
3804
3831
|
tagadds.append(tag)
|
|
3805
3832
|
tagperm = tuple(tag.split('.'))
|
|
3806
3833
|
runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
|
|
3807
3834
|
runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
|
|
3808
|
-
else:
|
|
3809
|
-
tags.append((len(tag), tag))
|
|
3810
3835
|
|
|
3811
|
-
|
|
3812
|
-
look = tag + '.'
|
|
3813
|
-
if any([tagadd.startswith(look) for tagadd in tagadds]):
|
|
3814
|
-
continue
|
|
3815
|
-
|
|
3816
|
-
tagadds.append(tag)
|
|
3817
|
-
tagperm = tuple(tag.split('.'))
|
|
3818
|
-
runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
|
|
3819
|
-
runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
|
|
3820
|
-
|
|
3821
|
-
for tag, tagdict in sode.get('tagprops', {}).items():
|
|
3822
|
-
for prop, (valu, stortype) in tagdict.items():
|
|
3836
|
+
for tag in sode.get('tagprops', {}).keys():
|
|
3823
3837
|
tagperm = tuple(tag.split('.'))
|
|
3824
3838
|
runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
|
|
3825
3839
|
runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
|
|
3826
3840
|
|
|
3827
|
-
|
|
3828
|
-
runt.
|
|
3829
|
-
|
|
3841
|
+
if not allows['ndata']:
|
|
3842
|
+
async for name in runt.snap.view.layers[0].iterNodeDataKeys(node.buid):
|
|
3843
|
+
runt.confirm(('node', 'data', 'pop', name), gateiden=layr0)
|
|
3844
|
+
runt.confirm(('node', 'data', 'set', name), gateiden=layr1)
|
|
3830
3845
|
|
|
3831
|
-
|
|
3832
|
-
verb
|
|
3833
|
-
|
|
3834
|
-
|
|
3846
|
+
if not allows['edges']:
|
|
3847
|
+
async for verb in runt.snap.view.layers[0].iterNodeEdgeVerbsN1(node.buid):
|
|
3848
|
+
runt.confirm(('node', 'edge', 'del', verb), gateiden=layr0)
|
|
3849
|
+
runt.confirm(('node', 'edge', 'add', verb), gateiden=layr1)
|
|
3835
3850
|
|
|
3836
3851
|
async def execStormCmd(self, runt, genr):
|
|
3837
3852
|
|
|
@@ -3841,12 +3856,31 @@ class MergeCmd(Cmd):
|
|
|
3841
3856
|
|
|
3842
3857
|
notags = self.opts.no_tags
|
|
3843
3858
|
onlytags = self.opts.only_tags
|
|
3859
|
+
doapply = self.opts.apply
|
|
3844
3860
|
|
|
3845
3861
|
tagfilter = self._getTagFilter()
|
|
3846
3862
|
propfilter = self._getPropFilter()
|
|
3847
3863
|
|
|
3848
|
-
layr0 = runt.snap.view.layers[0]
|
|
3849
|
-
layr1 = runt.snap.view.layers[1]
|
|
3864
|
+
layr0 = runt.snap.view.layers[0]
|
|
3865
|
+
layr1 = runt.snap.view.layers[1]
|
|
3866
|
+
|
|
3867
|
+
doperms = doapply and not (runt.isAdmin(gateiden=layr0.iden) and runt.isAdmin(gateiden=layr1.iden))
|
|
3868
|
+
|
|
3869
|
+
if doperms:
|
|
3870
|
+
allows = {
|
|
3871
|
+
'forms': runt.user.allowed(('node', 'del'), gateiden=layr0.iden, deepdeny=True) and
|
|
3872
|
+
runt.user.allowed(('node', 'add'), gateiden=layr1.iden, deepdeny=True),
|
|
3873
|
+
'props': runt.user.allowed(('node', 'prop', 'del'), gateiden=layr0.iden, deepdeny=True) and
|
|
3874
|
+
runt.user.allowed(('node', 'prop', 'set'), gateiden=layr1.iden, deepdeny=True),
|
|
3875
|
+
'tags': runt.user.allowed(('node', 'tag', 'del'), gateiden=layr0.iden, deepdeny=True) and
|
|
3876
|
+
runt.user.allowed(('node', 'tag', 'add'), gateiden=layr1.iden, deepdeny=True),
|
|
3877
|
+
'ndata': runt.user.allowed(('node', 'data', 'pop'), gateiden=layr0.iden, deepdeny=True) and
|
|
3878
|
+
runt.user.allowed(('node', 'data', 'set'), gateiden=layr1.iden, deepdeny=True),
|
|
3879
|
+
'edges': runt.user.allowed(('node', 'edge', 'del'), gateiden=layr0.iden, deepdeny=True) and
|
|
3880
|
+
runt.user.allowed(('node', 'edge', 'add'), gateiden=layr1.iden, deepdeny=True),
|
|
3881
|
+
}
|
|
3882
|
+
|
|
3883
|
+
doperms = not all(allows.values())
|
|
3850
3884
|
|
|
3851
3885
|
if self.opts.diff:
|
|
3852
3886
|
|
|
@@ -3854,7 +3888,7 @@ class MergeCmd(Cmd):
|
|
|
3854
3888
|
yield node, path
|
|
3855
3889
|
|
|
3856
3890
|
async def diffgenr():
|
|
3857
|
-
async for buid, sode in
|
|
3891
|
+
async for buid, sode in layr0.getStorNodes():
|
|
3858
3892
|
node = await runt.snap.getNodeByBuid(buid)
|
|
3859
3893
|
if node is not None:
|
|
3860
3894
|
yield node, runt.initPath(node)
|
|
@@ -3873,33 +3907,19 @@ class MergeCmd(Cmd):
|
|
|
3873
3907
|
sodes = await node.getStorNodes()
|
|
3874
3908
|
sode = sodes[0]
|
|
3875
3909
|
|
|
3876
|
-
if
|
|
3910
|
+
if doapply:
|
|
3877
3911
|
editor = s_snap.SnapEditor(snap)
|
|
3878
3912
|
|
|
3879
3913
|
subs = []
|
|
3880
3914
|
|
|
3881
|
-
async def sync():
|
|
3882
|
-
|
|
3883
|
-
if not self.opts.apply:
|
|
3884
|
-
subs.clear()
|
|
3885
|
-
return
|
|
3886
|
-
|
|
3887
|
-
addedits = editor.getNodeEdits()
|
|
3888
|
-
if addedits:
|
|
3889
|
-
await runt.snap.view.parent.storNodeEdits(addedits, meta=meta)
|
|
3890
|
-
|
|
3891
|
-
if subs:
|
|
3892
|
-
subedits = [(node.buid, node.form.name, subs)]
|
|
3893
|
-
await runt.snap.view.storNodeEdits(subedits, meta=meta)
|
|
3894
|
-
subs.clear()
|
|
3895
|
-
|
|
3896
3915
|
# check all node perms first
|
|
3897
|
-
if
|
|
3898
|
-
await self._checkNodePerms(node, sode, runt)
|
|
3916
|
+
if doperms:
|
|
3917
|
+
await self._checkNodePerms(node, sode, runt, allows)
|
|
3899
3918
|
|
|
3900
3919
|
form = node.form.name
|
|
3901
3920
|
if form == 'syn:tag':
|
|
3902
3921
|
if notags:
|
|
3922
|
+
await asyncio.sleep(0)
|
|
3903
3923
|
continue
|
|
3904
3924
|
else:
|
|
3905
3925
|
# avoid merging a tag if the node won't exist below us
|
|
@@ -3908,6 +3928,7 @@ class MergeCmd(Cmd):
|
|
|
3908
3928
|
if undr.get('valu') is not None:
|
|
3909
3929
|
break
|
|
3910
3930
|
else:
|
|
3931
|
+
await asyncio.sleep(0)
|
|
3911
3932
|
continue
|
|
3912
3933
|
|
|
3913
3934
|
protonode = None
|
|
@@ -3916,23 +3937,24 @@ class MergeCmd(Cmd):
|
|
|
3916
3937
|
valu = sode.get('valu')
|
|
3917
3938
|
if valu is not None:
|
|
3918
3939
|
|
|
3919
|
-
if tagfilter and form == 'syn:tag' and tagfilter(valu[0]):
|
|
3940
|
+
if tagfilter is not None and form == 'syn:tag' and tagfilter(valu[0]):
|
|
3941
|
+
await asyncio.sleep(0)
|
|
3920
3942
|
continue
|
|
3921
3943
|
|
|
3922
|
-
if not
|
|
3944
|
+
if not doapply:
|
|
3923
3945
|
valurepr = node.form.type.repr(valu[0])
|
|
3924
3946
|
await runt.printf(f'{nodeiden} {form} = {valurepr}')
|
|
3925
3947
|
else:
|
|
3926
3948
|
delnode = True
|
|
3927
3949
|
protonode = await editor.addNode(form, valu[0])
|
|
3928
3950
|
|
|
3929
|
-
elif
|
|
3951
|
+
elif doapply:
|
|
3930
3952
|
protonode = await editor.addNode(form, node.ndef[1], norminfo={})
|
|
3931
3953
|
|
|
3932
3954
|
for name, (valu, stortype) in sode.get('props', {}).items():
|
|
3933
3955
|
|
|
3934
3956
|
prop = node.form.prop(name)
|
|
3935
|
-
if propfilter:
|
|
3957
|
+
if propfilter is not None:
|
|
3936
3958
|
if name[0] == '.':
|
|
3937
3959
|
if propfilter(name):
|
|
3938
3960
|
continue
|
|
@@ -3942,7 +3964,7 @@ class MergeCmd(Cmd):
|
|
|
3942
3964
|
|
|
3943
3965
|
if prop.info.get('ro'):
|
|
3944
3966
|
if name == '.created':
|
|
3945
|
-
if
|
|
3967
|
+
if doapply:
|
|
3946
3968
|
protonode.props['.created'] = valu
|
|
3947
3969
|
subs.append((s_layer.EDIT_PROP_DEL, (name, valu, stortype), ()))
|
|
3948
3970
|
continue
|
|
@@ -3952,8 +3974,8 @@ class MergeCmd(Cmd):
|
|
|
3952
3974
|
props = undr.get('props')
|
|
3953
3975
|
if props is not None:
|
|
3954
3976
|
curv = props.get(name)
|
|
3955
|
-
if curv is not None
|
|
3956
|
-
isset =
|
|
3977
|
+
if curv is not None:
|
|
3978
|
+
isset = curv[0] != valu
|
|
3957
3979
|
break
|
|
3958
3980
|
|
|
3959
3981
|
if isset:
|
|
@@ -3963,24 +3985,23 @@ class MergeCmd(Cmd):
|
|
|
3963
3985
|
await runt.snap.warn(mesg)
|
|
3964
3986
|
continue
|
|
3965
3987
|
|
|
3966
|
-
if not
|
|
3988
|
+
if not doapply:
|
|
3967
3989
|
valurepr = prop.type.repr(valu)
|
|
3968
3990
|
await runt.printf(f'{nodeiden} {form}:{name} = {valurepr}')
|
|
3969
3991
|
else:
|
|
3970
3992
|
await protonode.set(name, valu)
|
|
3971
3993
|
subs.append((s_layer.EDIT_PROP_DEL, (name, valu, stortype), ()))
|
|
3972
3994
|
|
|
3973
|
-
if
|
|
3995
|
+
if doapply and protonode is None:
|
|
3974
3996
|
protonode = await editor.addNode(form, node.ndef[1], norminfo={})
|
|
3975
3997
|
|
|
3976
3998
|
if not notags:
|
|
3977
3999
|
for tag, valu in sode.get('tags', {}).items():
|
|
3978
4000
|
|
|
3979
|
-
if tagfilter and tagfilter(tag):
|
|
4001
|
+
if tagfilter is not None and tagfilter(tag):
|
|
3980
4002
|
continue
|
|
3981
4003
|
|
|
3982
|
-
|
|
3983
|
-
if not self.opts.apply:
|
|
4004
|
+
if not doapply:
|
|
3984
4005
|
valurepr = ''
|
|
3985
4006
|
if valu != (None, None):
|
|
3986
4007
|
tagrepr = runt.model.type('ival').repr(valu)
|
|
@@ -3992,12 +4013,11 @@ class MergeCmd(Cmd):
|
|
|
3992
4013
|
|
|
3993
4014
|
for tag, tagdict in sode.get('tagprops', {}).items():
|
|
3994
4015
|
|
|
3995
|
-
if tagfilter and tagfilter(tag):
|
|
4016
|
+
if tagfilter is not None and tagfilter(tag):
|
|
3996
4017
|
continue
|
|
3997
4018
|
|
|
3998
4019
|
for prop, (valu, stortype) in tagdict.items():
|
|
3999
|
-
|
|
4000
|
-
if not self.opts.apply:
|
|
4020
|
+
if not doapply:
|
|
4001
4021
|
valurepr = repr(valu)
|
|
4002
4022
|
await runt.printf(f'{nodeiden} {form}#{tag}:{prop} = {valurepr}')
|
|
4003
4023
|
else:
|
|
@@ -4006,31 +4026,33 @@ class MergeCmd(Cmd):
|
|
|
4006
4026
|
|
|
4007
4027
|
if not onlytags or form == 'syn:tag':
|
|
4008
4028
|
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
if not self.opts.apply:
|
|
4029
|
+
async for name, valu in s_coro.pause(layr0.iterNodeData(node.buid)):
|
|
4030
|
+
if not doapply:
|
|
4012
4031
|
valurepr = repr(valu)
|
|
4013
4032
|
await runt.printf(f'{nodeiden} {form} DATA {name} = {valurepr}')
|
|
4014
4033
|
else:
|
|
4015
4034
|
await protonode.setData(name, valu)
|
|
4016
4035
|
subs.append((s_layer.EDIT_NODEDATA_DEL, (name, valu), ()))
|
|
4017
|
-
if len(subs) >= 1000:
|
|
4018
|
-
await asyncio.sleep(0)
|
|
4019
4036
|
|
|
4020
|
-
async for edge in
|
|
4037
|
+
async for edge in s_coro.pause(layr0.iterNodeEdgesN1(node.buid)):
|
|
4021
4038
|
name, dest = edge
|
|
4022
|
-
if not
|
|
4039
|
+
if not doapply:
|
|
4023
4040
|
await runt.printf(f'{nodeiden} {form} +({name})> {dest}')
|
|
4024
4041
|
else:
|
|
4025
4042
|
await protonode.addEdge(name, dest)
|
|
4026
4043
|
subs.append((s_layer.EDIT_EDGE_DEL, edge, ()))
|
|
4027
|
-
if len(subs) >= 1000:
|
|
4028
|
-
await asyncio.sleep(0)
|
|
4029
4044
|
|
|
4030
4045
|
if delnode:
|
|
4031
4046
|
subs.append((s_layer.EDIT_NODE_DEL, valu, ()))
|
|
4032
4047
|
|
|
4033
|
-
|
|
4048
|
+
if doapply:
|
|
4049
|
+
addedits = editor.getNodeEdits()
|
|
4050
|
+
if addedits:
|
|
4051
|
+
await runt.snap.view.parent.storNodeEdits(addedits, meta=meta)
|
|
4052
|
+
|
|
4053
|
+
if subs:
|
|
4054
|
+
subedits = [(node.buid, node.form.name, subs)]
|
|
4055
|
+
await runt.snap.view.storNodeEdits(subedits, meta=meta)
|
|
4034
4056
|
|
|
4035
4057
|
runt.snap.clearCachedNode(node.buid)
|
|
4036
4058
|
yield await runt.snap.getNodeByBuid(node.buid), path
|
synapse/lib/stormlib/storm.py
CHANGED
|
@@ -1,18 +1,102 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
import synapse.exc as s_exc
|
|
4
|
+
import synapse.common as s_common
|
|
4
5
|
|
|
6
|
+
import synapse.lib.storm as s_storm
|
|
5
7
|
import synapse.lib.stormtypes as s_stormtypes
|
|
6
8
|
|
|
7
9
|
evaldesc = '''\
|
|
8
|
-
Evaluate a
|
|
10
|
+
Evaluate a Storm runtime value and optionally cast/coerce it.
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
Note:
|
|
13
|
+
If Storm logging is enabled, the expression being evaluated will be logged
|
|
14
|
+
separately.
|
|
15
|
+
'''
|
|
16
|
+
|
|
17
|
+
rundesc = '''
|
|
18
|
+
Run a Storm query and yield the messages output by the Storm interpreter.
|
|
19
|
+
|
|
20
|
+
Note:
|
|
21
|
+
If Storm logging is enabled, the query being run will be logged separately.
|
|
12
22
|
'''
|
|
13
23
|
|
|
14
24
|
stormlogger = logging.getLogger('synapse.storm')
|
|
15
25
|
|
|
26
|
+
class StormExecCmd(s_storm.Cmd):
|
|
27
|
+
'''
|
|
28
|
+
Execute text or an embedded query object as Storm in the current pipeline.
|
|
29
|
+
|
|
30
|
+
NOTE: It is recommended to avoid using this where possible to avoid potential
|
|
31
|
+
query injection risks. If you must use this, take care to ensure any values
|
|
32
|
+
being executed have been properly sanitized.
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
|
|
36
|
+
// Add nodes using text in a variable
|
|
37
|
+
$query = '[ inet:fqdn=foo.com inet:fqdn=bar.net ]'
|
|
38
|
+
storm.exec $query
|
|
39
|
+
|
|
40
|
+
// Filter nodes in the pipeline using text in a variable
|
|
41
|
+
$filter = '-:asn=10'
|
|
42
|
+
inet:ipv4:asn
|
|
43
|
+
storm.exec $filter
|
|
44
|
+
|
|
45
|
+
// Pivot using an embedded query
|
|
46
|
+
$pivot = ${ -> inet:asn }
|
|
47
|
+
inet:ipv4:asn
|
|
48
|
+
storm.exec $pivot
|
|
49
|
+
'''
|
|
50
|
+
name = 'storm.exec'
|
|
51
|
+
def getArgParser(self):
|
|
52
|
+
pars = s_storm.Cmd.getArgParser(self)
|
|
53
|
+
pars.add_argument('query', help='The Storm to execute.')
|
|
54
|
+
return pars
|
|
55
|
+
|
|
56
|
+
async def execStormCmd(self, runt, genr):
|
|
57
|
+
|
|
58
|
+
if self.runtsafe:
|
|
59
|
+
|
|
60
|
+
text = await s_stormtypes.tostr(self.opts.query)
|
|
61
|
+
query = await runt.getStormQuery(text)
|
|
62
|
+
|
|
63
|
+
extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
|
|
64
|
+
stormlogger.info(f'Executing storm query via storm.exec {{{text}}} as [{self.runt.user.name}]', extra=extra)
|
|
65
|
+
|
|
66
|
+
async with runt.getSubRuntime(query) as subr:
|
|
67
|
+
async for subp in subr.execute(genr=genr):
|
|
68
|
+
yield subp
|
|
69
|
+
|
|
70
|
+
else:
|
|
71
|
+
|
|
72
|
+
item = None
|
|
73
|
+
async for item in genr:
|
|
74
|
+
break
|
|
75
|
+
|
|
76
|
+
text = await s_stormtypes.tostr(self.opts.query)
|
|
77
|
+
query = await runt.getStormQuery(text)
|
|
78
|
+
|
|
79
|
+
extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
|
|
80
|
+
stormlogger.info(f'Executing storm query via storm.exec {{{text}}} as [{self.runt.user.name}]', extra=extra)
|
|
81
|
+
|
|
82
|
+
async with runt.getSubRuntime(query) as subr:
|
|
83
|
+
async for subp in subr.execute(genr=s_common.agen(item)):
|
|
84
|
+
yield subp
|
|
85
|
+
|
|
86
|
+
async for item in genr:
|
|
87
|
+
text = await s_stormtypes.tostr(self.opts.query)
|
|
88
|
+
query = await runt.getStormQuery(text)
|
|
89
|
+
|
|
90
|
+
subr.runtvars.clear()
|
|
91
|
+
subr.query = query
|
|
92
|
+
subr._initRuntVars(query)
|
|
93
|
+
|
|
94
|
+
extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
|
|
95
|
+
stormlogger.info(f'Executing storm query via storm.exec {{{text}}} as [{self.runt.user.name}]', extra=extra)
|
|
96
|
+
|
|
97
|
+
async for subp in subr.execute(genr=s_common.agen(item)):
|
|
98
|
+
yield subp
|
|
99
|
+
|
|
16
100
|
@s_stormtypes.registry.registerLib
|
|
17
101
|
class LibStorm(s_stormtypes.Lib):
|
|
18
102
|
'''
|
|
@@ -25,15 +109,43 @@ class LibStorm(s_stormtypes.Lib):
|
|
|
25
109
|
{'name': 'text', 'type': 'str', 'desc': 'A storm expression string.'},
|
|
26
110
|
{'name': 'cast', 'type': 'str', 'desc': 'A type to cast the result to.', 'default': None},
|
|
27
111
|
),
|
|
28
|
-
'returns': {'type': 'any', 'desc': 'The value of the expression and optional cast.'
|
|
112
|
+
'returns': {'type': 'any', 'desc': 'The value of the expression and optional cast.'}}},
|
|
113
|
+
{'name': 'run', 'desc': rundesc,
|
|
114
|
+
'type': {'type': 'function', '_funcname': '_runStorm',
|
|
115
|
+
'args': (
|
|
116
|
+
{'name': 'query', 'type': 'str', 'desc': 'A Storm query string.'},
|
|
117
|
+
{'name': 'opts', 'type': 'dict', 'desc': 'Storm options dictionary.', 'default': None},
|
|
118
|
+
),
|
|
119
|
+
'returns': {'name': 'yields', 'type': 'list', 'desc': 'The output messages from the Storm runtime.'}}},
|
|
29
120
|
)
|
|
30
121
|
_storm_lib_path = ('storm',)
|
|
31
122
|
|
|
32
123
|
def getObjLocals(self):
|
|
33
124
|
return {
|
|
125
|
+
'run': self._runStorm,
|
|
34
126
|
'eval': self._evalStorm,
|
|
35
127
|
}
|
|
36
128
|
|
|
129
|
+
async def _runStorm(self, query, opts=None):
|
|
130
|
+
|
|
131
|
+
opts = await s_stormtypes.toprim(opts)
|
|
132
|
+
query = await s_stormtypes.tostr(query)
|
|
133
|
+
|
|
134
|
+
if opts is None:
|
|
135
|
+
opts = {}
|
|
136
|
+
|
|
137
|
+
user = opts.get('user')
|
|
138
|
+
if user is None:
|
|
139
|
+
user = opts['user'] = self.runt.user.iden
|
|
140
|
+
|
|
141
|
+
if user != self.runt.user.iden:
|
|
142
|
+
self.runt.confirm(('impersonate',))
|
|
143
|
+
|
|
144
|
+
opts.setdefault('view', self.runt.snap.view.iden)
|
|
145
|
+
|
|
146
|
+
async for mesg in self.runt.snap.view.core.storm(query, opts=opts):
|
|
147
|
+
yield mesg
|
|
148
|
+
|
|
37
149
|
@s_stormtypes.stormfunc(readonly=True)
|
|
38
150
|
async def _evalStorm(self, text, cast=None):
|
|
39
151
|
|
|
@@ -41,7 +153,7 @@ class LibStorm(s_stormtypes.Lib):
|
|
|
41
153
|
cast = await s_stormtypes.tostr(cast, noneok=True)
|
|
42
154
|
|
|
43
155
|
if self.runt.snap.core.stormlog:
|
|
44
|
-
extra = await self.runt.snap.core.getLogExtra(text=text)
|
|
156
|
+
extra = await self.runt.snap.core.getLogExtra(text=text, view=self.runt.snap.view.iden)
|
|
45
157
|
stormlogger.info(f'Executing storm query via $lib.storm.eval() {{{text}}} as [{self.runt.user.name}]', extra=extra)
|
|
46
158
|
|
|
47
159
|
casttype = None
|
synapse/lib/types.py
CHANGED
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 176, 0)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = '16ee721a6b7221344eaf946c3ab4602dda546b1a'
|
synapse/models/inet.py
CHANGED
|
@@ -1519,6 +1519,7 @@ class InetModule(s_module.CoreModule):
|
|
|
1519
1519
|
'doc': 'An object status enumeration.'}),
|
|
1520
1520
|
|
|
1521
1521
|
('inet:service:account', ('guid', {}), {
|
|
1522
|
+
'interfaces': ('inet:service:object',),
|
|
1522
1523
|
'doc': 'An account within a service platform. Accounts may be instance specific.'}),
|
|
1523
1524
|
|
|
1524
1525
|
('inet:service:permission:type:taxonomy', ('taxonomy', {}), {
|
|
@@ -1557,6 +1558,10 @@ class InetModule(s_module.CoreModule):
|
|
|
1557
1558
|
'interfaces': ('inet:service:object',),
|
|
1558
1559
|
'doc': 'A channel used to distribute messages.'}),
|
|
1559
1560
|
|
|
1561
|
+
('inet:service:thread', ('guid', {}), {
|
|
1562
|
+
'interfaces': ('inet:service:object',),
|
|
1563
|
+
'doc': 'A message thread.'}),
|
|
1564
|
+
|
|
1560
1565
|
('inet:service:channel:member', ('guid', {}), {
|
|
1561
1566
|
'interfaces': ('inet:service:object',),
|
|
1562
1567
|
'doc': 'Represents a service account being a member of a channel.'}),
|
|
@@ -3657,9 +3662,15 @@ class InetModule(s_module.CoreModule):
|
|
|
3657
3662
|
('channel', ('inet:service:channel', {}), {
|
|
3658
3663
|
'doc': 'The channel that the message was sent to.'}),
|
|
3659
3664
|
|
|
3665
|
+
('thread', ('inet:service:thread', {}), {
|
|
3666
|
+
'doc': 'The thread which contains the message.'}),
|
|
3667
|
+
|
|
3660
3668
|
('public', ('bool', {}), {
|
|
3661
3669
|
'doc': 'Set to true if the message is publicly visible.'}),
|
|
3662
3670
|
|
|
3671
|
+
('title', ('str', {'lower': True, 'onespace': True}), {
|
|
3672
|
+
'doc': 'The message title.'}),
|
|
3673
|
+
|
|
3663
3674
|
('text', ('str', {}), {
|
|
3664
3675
|
'disp': {'hint': 'text'},
|
|
3665
3676
|
'doc': 'The text body of the message.'}),
|
|
@@ -3725,6 +3736,18 @@ class InetModule(s_module.CoreModule):
|
|
|
3725
3736
|
'doc': 'The time period where the channel was available.'}),
|
|
3726
3737
|
)),
|
|
3727
3738
|
|
|
3739
|
+
('inet:service:thread', {}, (
|
|
3740
|
+
|
|
3741
|
+
('title', ('str', {'lower': True, 'onespace': True}), {
|
|
3742
|
+
'doc': 'The title of the thread.'}),
|
|
3743
|
+
|
|
3744
|
+
('channel', ('inet:service:channel', {}), {
|
|
3745
|
+
'doc': 'The channel that contains the thread.'}),
|
|
3746
|
+
|
|
3747
|
+
('message', ('inet:service:message', {}), {
|
|
3748
|
+
'doc': 'The message which initiated the thread.'}),
|
|
3749
|
+
)),
|
|
3750
|
+
|
|
3728
3751
|
('inet:service:channel:member', {}, (
|
|
3729
3752
|
|
|
3730
3753
|
('channel', ('inet:service:channel', {}), {
|