synapse 2.220.0__py311-none-any.whl → 2.222.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +34 -14
- synapse/data/lark/storm.lark +9 -6
- synapse/lib/ast.py +8 -2
- synapse/lib/layer.py +149 -8
- synapse/lib/parser.py +1 -0
- synapse/lib/rstorm.py +83 -2
- synapse/lib/schemas.py +4 -0
- synapse/lib/snap.py +21 -13
- synapse/lib/stormhttp.py +10 -10
- synapse/lib/stormlib/aha.py +3 -3
- synapse/lib/stormlib/auth.py +11 -11
- synapse/lib/stormlib/cell.py +1 -1
- synapse/lib/stormlib/cortex.py +10 -10
- synapse/lib/stormlib/env.py +4 -5
- synapse/lib/stormlib/ethereum.py +1 -1
- synapse/lib/stormlib/gen.py +3 -3
- synapse/lib/stormlib/hex.py +2 -2
- synapse/lib/stormlib/imap.py +2 -2
- synapse/lib/stormlib/infosec.py +2 -2
- synapse/lib/stormlib/iters.py +2 -2
- synapse/lib/stormlib/model.py +5 -5
- synapse/lib/stormlib/notifications.py +1 -1
- synapse/lib/stormlib/oauth.py +2 -2
- synapse/lib/stormlib/project.py +3 -3
- synapse/lib/stormlib/scrape.py +2 -1
- synapse/lib/stormlib/smtp.py +3 -3
- synapse/lib/stormlib/stats.py +2 -2
- synapse/lib/stormlib/stix.py +2 -2
- synapse/lib/stormlib/utils.py +19 -0
- synapse/lib/stormlib/vault.py +1 -1
- synapse/lib/stormlib/xml.py +2 -2
- synapse/lib/stormlib/yaml.py +1 -1
- synapse/lib/stormtypes.py +182 -64
- synapse/lib/version.py +2 -2
- synapse/models/orgs.py +3 -0
- synapse/tests/test_lib_grammar.py +4 -4
- synapse/tests/test_lib_layer.py +86 -67
- synapse/tests/test_lib_rstorm.py +180 -0
- synapse/tests/test_lib_storm.py +80 -1
- synapse/tests/test_lib_stormlib_auth.py +84 -0
- synapse/tests/test_lib_stormlib_cortex.py +1 -0
- synapse/tests/test_lib_stormlib_env.py +3 -1
- synapse/tests/test_lib_stormlib_utils.py +10 -0
- synapse/tests/test_lib_stormtypes.py +576 -2
- synapse/tests/test_model_orgs.py +6 -1
- synapse/tools/aha/list.py +9 -9
- synapse/tools/aha/provision/service.py +2 -2
- {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/METADATA +1 -1
- {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/RECORD +52 -52
- {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/WHEEL +0 -0
- {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/licenses/LICENSE +0 -0
- {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py
CHANGED
|
@@ -624,7 +624,8 @@ class CoreApi(s_cell.CellApi):
|
|
|
624
624
|
Returns:
|
|
625
625
|
AsyncIterator[Tuple(buid, valu)]
|
|
626
626
|
'''
|
|
627
|
-
self.user.
|
|
627
|
+
if not self.user.allowed(('layer', 'lift', layriden)):
|
|
628
|
+
self.user.confirm(('layer', 'read', layriden))
|
|
628
629
|
async for item in self.cell.iterFormRows(layriden, form, stortype=stortype, startvalu=startvalu):
|
|
629
630
|
yield item
|
|
630
631
|
|
|
@@ -642,7 +643,8 @@ class CoreApi(s_cell.CellApi):
|
|
|
642
643
|
Returns:
|
|
643
644
|
AsyncIterator[Tuple(buid, valu)]
|
|
644
645
|
'''
|
|
645
|
-
self.user.
|
|
646
|
+
if not self.user.allowed(('layer', 'lift', layriden)):
|
|
647
|
+
self.user.confirm(('layer', 'read', layriden))
|
|
646
648
|
async for item in self.cell.iterPropRows(layriden, form, prop, stortype=stortype, startvalu=startvalu):
|
|
647
649
|
yield item
|
|
648
650
|
|
|
@@ -659,7 +661,8 @@ class CoreApi(s_cell.CellApi):
|
|
|
659
661
|
Returns:
|
|
660
662
|
AsyncIterator[Tuple(buid, valu)]
|
|
661
663
|
'''
|
|
662
|
-
self.user.
|
|
664
|
+
if not self.user.allowed(('layer', 'lift', layriden)):
|
|
665
|
+
self.user.confirm(('layer', 'read', layriden))
|
|
663
666
|
async for item in self.cell.iterUnivRows(layriden, prop, stortype=stortype, startvalu=startvalu):
|
|
664
667
|
yield item
|
|
665
668
|
|
|
@@ -680,7 +683,8 @@ class CoreApi(s_cell.CellApi):
|
|
|
680
683
|
This yields (buid, (tagvalu, form)) instead of just buid, valu in order to allow resuming an interrupted
|
|
681
684
|
call by feeding the last value retrieved into starttupl
|
|
682
685
|
'''
|
|
683
|
-
self.user.
|
|
686
|
+
if not self.user.allowed(('layer', 'lift', layriden)):
|
|
687
|
+
self.user.confirm(('layer', 'read', layriden))
|
|
684
688
|
async for item in self.cell.iterTagRows(layriden, tag, form=form, starttupl=starttupl):
|
|
685
689
|
yield item
|
|
686
690
|
|
|
@@ -699,7 +703,8 @@ class CoreApi(s_cell.CellApi):
|
|
|
699
703
|
Returns:
|
|
700
704
|
AsyncIterator[Tuple(buid, valu)]
|
|
701
705
|
'''
|
|
702
|
-
self.user.
|
|
706
|
+
if not self.user.allowed(('layer', 'lift', layriden)):
|
|
707
|
+
self.user.confirm(('layer', 'read', layriden))
|
|
703
708
|
async for item in self.cell.iterTagPropRows(layriden, tag, prop, form=form, stortype=stortype,
|
|
704
709
|
startvalu=startvalu):
|
|
705
710
|
yield item
|
|
@@ -1366,6 +1371,30 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1366
1371
|
|
|
1367
1372
|
def _initCorePerms(self):
|
|
1368
1373
|
self._cortex_permdefs.extend((
|
|
1374
|
+
{'perm': ('axon', 'upload'), 'gate': 'cortex',
|
|
1375
|
+
'desc': 'Controls the ability to upload a file to the Axon.'},
|
|
1376
|
+
{'perm': ('axon', 'get'), 'gate': 'cortex',
|
|
1377
|
+
'desc': 'Controls the ability to retrieve a file from the Axon.'},
|
|
1378
|
+
{'perm': ('axon', 'has'), 'gate': 'cortex',
|
|
1379
|
+
'desc': 'Controls the ability to check if the Axon contains a file.'},
|
|
1380
|
+
{'perm': ('axon', 'del'), 'gate': 'cortex',
|
|
1381
|
+
'desc': 'Controls the ability to remove a file from the Axon.'},
|
|
1382
|
+
|
|
1383
|
+
{'perm': ('layer', 'add'), 'gate': 'cortex',
|
|
1384
|
+
'desc': 'Controls the ability to add Layers to the cortex.'},
|
|
1385
|
+
{'perm': ('layer', 'del'), 'gate': 'cortex',
|
|
1386
|
+
'desc': 'Controls the ability to remove Layers from the cortex.'},
|
|
1387
|
+
{'perm': ('layer', 'read'), 'gate': 'layer',
|
|
1388
|
+
'desc': 'Controls the ability to read/lift from a Layer.'},
|
|
1389
|
+
{'perm': ('layer', 'read', '<iden>'), 'gate': 'cortex',
|
|
1390
|
+
'desc': 'Controls the ability to read/lift from a specific Layer.'},
|
|
1391
|
+
{'perm': ('layer', 'set', '<name>'), 'gate': 'layer',
|
|
1392
|
+
'desc': 'Controls the ability to configure properties of a Layer.'},
|
|
1393
|
+
{'perm': ('layer', 'write'), 'gate': 'layer',
|
|
1394
|
+
'desc': 'Controls the ability to write to a Layer.'},
|
|
1395
|
+
{'perm': ('layer', 'write', '<iden>'), 'gate': 'cortex',
|
|
1396
|
+
'desc': 'Controls the ability to write to a specific Layer.'},
|
|
1397
|
+
|
|
1369
1398
|
{'perm': ('model', 'form', 'add'), 'gate': 'cortex',
|
|
1370
1399
|
'desc': 'Controls access to adding extended model forms.'},
|
|
1371
1400
|
{'perm': ('model', 'form', 'add', '<form>'), 'gate': 'cortex',
|
|
@@ -1519,15 +1548,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1519
1548
|
{'perm': ('view', 'set', '<setting>'), 'gate': 'view',
|
|
1520
1549
|
'desc': 'Controls access to change view settings.',
|
|
1521
1550
|
'ex': 'view.set.name'},
|
|
1522
|
-
|
|
1523
|
-
{'perm': ('axon', 'upload'), 'gate': 'cortex',
|
|
1524
|
-
'desc': 'Controls the ability to upload a file to the Axon.'},
|
|
1525
|
-
{'perm': ('axon', 'get'), 'gate': 'cortex',
|
|
1526
|
-
'desc': 'Controls the ability to retrieve a file from the Axon.'},
|
|
1527
|
-
{'perm': ('axon', 'has'), 'gate': 'cortex',
|
|
1528
|
-
'desc': 'Controls the ability to check if the Axon contains a file.'},
|
|
1529
|
-
{'perm': ('axon', 'del'), 'gate': 'cortex',
|
|
1530
|
-
'desc': 'Controls the ability to remove a file from the Axon.'},
|
|
1531
1551
|
))
|
|
1532
1552
|
for pdef in self._cortex_permdefs:
|
|
1533
1553
|
s_schemas.reqValidPermDef(pdef)
|
synapse/data/lark/storm.lark
CHANGED
|
@@ -238,10 +238,10 @@ liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | l
|
|
|
238
238
|
|
|
239
239
|
_DEREF.3: /\*(?=\$)/
|
|
240
240
|
|
|
241
|
-
liftformtag: (
|
|
242
|
-
liftpropby: (
|
|
243
|
-
liftprop: (
|
|
244
|
-
liftbyarray: (
|
|
241
|
+
liftformtag: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname [_cmpr _valu]
|
|
242
|
+
liftpropby: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) _cmpr _valu
|
|
243
|
+
liftprop: (PROPS | UNIVNAME | WILDPROPS | derefprops)
|
|
244
|
+
liftbyarray: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) "*[" _safe_cmpr _valu "]"
|
|
245
245
|
lifttagtag: (_HASH | _HASHSPACE) tagname [_cmpr _valu]
|
|
246
246
|
liftbytag: (tagname | tagnamewithspace) [_cmpr _valu]
|
|
247
247
|
liftbytagprop: (tagprop | tagpropwithspace) [_cmpr _valu]
|
|
@@ -249,7 +249,7 @@ liftbyformtagprop: formtagprop [_cmpr _valu]
|
|
|
249
249
|
|
|
250
250
|
tagprop: tagname _COLONNOSPACE (BASEPROP | _varvalu)
|
|
251
251
|
tagpropwithspace: tagnamewithspace _COLONNOSPACE (BASEPROP | _varvalu) -> tagprop
|
|
252
|
-
formtagprop: (
|
|
252
|
+
formtagprop: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname _COLONNOSPACE (BASEPROP | _varvalu)
|
|
253
253
|
_COLONNOSPACE.2: /(?<!\s):/
|
|
254
254
|
|
|
255
255
|
_funcarg: (VARTOKN [EQNOSPACE _valu])
|
|
@@ -484,7 +484,7 @@ EMBEDPROPS.2: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+((\:\:|\:|\.)[a-z0-9_]+)*(?![:.a-z0
|
|
|
484
484
|
UNIVNAME.2: /(?<=^|[\s\|\{\(\[+=-])\.[a-z_][a-z0-9_]*([:.][a-z0-9_]+)*/
|
|
485
485
|
univprop: UNIVNAME | "." _varvalu
|
|
486
486
|
// A full property or a universal property
|
|
487
|
-
formname: PROPS |
|
|
487
|
+
formname: PROPS | derefprops
|
|
488
488
|
// A relative property
|
|
489
489
|
relprop: RELNAME | _COLONDOLLAR _varvaluatom
|
|
490
490
|
|
|
@@ -494,6 +494,9 @@ RELNAME: /(?<!\w):\.?[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
|
|
|
494
494
|
// Similar to PROPS but does not require a colon
|
|
495
495
|
BASEPROP: /[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
|
|
496
496
|
|
|
497
|
+
// Derefed variable for prop names
|
|
498
|
+
derefprops: _DEREF _varvalu
|
|
499
|
+
|
|
497
500
|
// The entry point for a $(...) expression. The initial dollar sign is now optional
|
|
498
501
|
_dollarexprs: "$"? dollaroper
|
|
499
502
|
|
synapse/lib/ast.py
CHANGED
|
@@ -2,7 +2,6 @@ import types
|
|
|
2
2
|
import asyncio
|
|
3
3
|
import decimal
|
|
4
4
|
import fnmatch
|
|
5
|
-
import hashlib
|
|
6
5
|
import logging
|
|
7
6
|
import binascii
|
|
8
7
|
import itertools
|
|
@@ -1833,7 +1832,7 @@ class LiftProp(LiftOper):
|
|
|
1833
1832
|
|
|
1834
1833
|
assert len(self.kids) == 1
|
|
1835
1834
|
|
|
1836
|
-
name = await
|
|
1835
|
+
name = await self.kids[0].compute(runt, path)
|
|
1837
1836
|
|
|
1838
1837
|
prop = runt.model.props.get(name)
|
|
1839
1838
|
if prop is not None:
|
|
@@ -4029,6 +4028,13 @@ class FormName(Value):
|
|
|
4029
4028
|
async def compute(self, runt, path):
|
|
4030
4029
|
return await self.kids[0].compute(runt, path)
|
|
4031
4030
|
|
|
4031
|
+
class DerefProps(Value):
|
|
4032
|
+
async def compute(self, runt, path):
|
|
4033
|
+
valu = await toprim(await self.kids[0].compute(runt, path))
|
|
4034
|
+
if (exc := await s_stormtypes.typeerr(valu, str)) is not None:
|
|
4035
|
+
raise self.kids[0].addExcInfo(exc)
|
|
4036
|
+
return valu
|
|
4037
|
+
|
|
4032
4038
|
class RelProp(PropName):
|
|
4033
4039
|
pass
|
|
4034
4040
|
|
synapse/lib/layer.py
CHANGED
|
@@ -119,6 +119,7 @@ class LayerApi(s_cell.CellApi):
|
|
|
119
119
|
|
|
120
120
|
self.layr = layr
|
|
121
121
|
self.liftperm = ('layer', 'lift', self.layr.iden)
|
|
122
|
+
self.readperm = ('layer', 'read', self.layr.iden)
|
|
122
123
|
self.writeperm = ('layer', 'write', self.layr.iden)
|
|
123
124
|
|
|
124
125
|
async def iterLayerNodeEdits(self):
|
|
@@ -126,7 +127,8 @@ class LayerApi(s_cell.CellApi):
|
|
|
126
127
|
Scan the full layer and yield artificial nodeedit sets.
|
|
127
128
|
'''
|
|
128
129
|
|
|
129
|
-
|
|
130
|
+
if not self.allowed(self.liftperm):
|
|
131
|
+
await self._reqUserAllowed(self.readperm)
|
|
130
132
|
async for item in self.layr.iterLayerNodeEdits():
|
|
131
133
|
yield item
|
|
132
134
|
await asyncio.sleep(0)
|
|
@@ -165,13 +167,15 @@ class LayerApi(s_cell.CellApi):
|
|
|
165
167
|
|
|
166
168
|
Once caught up with storage, yield them in realtime.
|
|
167
169
|
'''
|
|
168
|
-
|
|
170
|
+
if not self.allowed(self.liftperm):
|
|
171
|
+
await self._reqUserAllowed(self.readperm)
|
|
169
172
|
async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
|
|
170
173
|
yield item
|
|
171
174
|
await asyncio.sleep(0)
|
|
172
175
|
|
|
173
176
|
async def syncNodeEdits2(self, offs, wait=True):
|
|
174
|
-
|
|
177
|
+
if not self.allowed(self.liftperm):
|
|
178
|
+
await self._reqUserAllowed(self.readperm)
|
|
175
179
|
async for item in self.layr.syncNodeEdits2(offs, wait=wait):
|
|
176
180
|
yield item
|
|
177
181
|
await asyncio.sleep(0)
|
|
@@ -180,18 +184,21 @@ class LayerApi(s_cell.CellApi):
|
|
|
180
184
|
'''
|
|
181
185
|
Returns what will be the *next* nodeedit log index.
|
|
182
186
|
'''
|
|
183
|
-
|
|
187
|
+
if not self.allowed(self.liftperm):
|
|
188
|
+
await self._reqUserAllowed(self.readperm)
|
|
184
189
|
return await self.layr.getEditIndx()
|
|
185
190
|
|
|
186
191
|
async def getEditSize(self):
|
|
187
192
|
'''
|
|
188
193
|
Return the total number of (edits, meta) pairs in the layer changelog.
|
|
189
194
|
'''
|
|
190
|
-
|
|
195
|
+
if not self.allowed(self.liftperm):
|
|
196
|
+
await self._reqUserAllowed(self.readperm)
|
|
191
197
|
return await self.layr.getEditSize()
|
|
192
198
|
|
|
193
199
|
async def getIden(self):
|
|
194
|
-
|
|
200
|
+
if not self.allowed(self.liftperm):
|
|
201
|
+
await self._reqUserAllowed(self.readperm)
|
|
195
202
|
return self.layr.iden
|
|
196
203
|
|
|
197
204
|
BUID_CACHE_SIZE = 10000
|
|
@@ -3350,7 +3357,99 @@ class Layer(s_nexus.Pusher):
|
|
|
3350
3357
|
nodeedits = [(buid, newp_formname, [set_edit])]
|
|
3351
3358
|
|
|
3352
3359
|
_, changes = await self.saveNodeEdits(nodeedits, meta)
|
|
3353
|
-
return
|
|
3360
|
+
return any(c[2] for c in changes)
|
|
3361
|
+
|
|
3362
|
+
async def delStorNode(self, buid, meta):
|
|
3363
|
+
'''
|
|
3364
|
+
Delete all node information in this layer.
|
|
3365
|
+
|
|
3366
|
+
Deletes props, tagprops, tags, n1edges, n2edges, nodedata, and node valu.
|
|
3367
|
+
'''
|
|
3368
|
+
sode = self._getStorNode(buid)
|
|
3369
|
+
if sode is None:
|
|
3370
|
+
return False
|
|
3371
|
+
|
|
3372
|
+
formname = sode.get('form')
|
|
3373
|
+
|
|
3374
|
+
edits = []
|
|
3375
|
+
nodeedits = []
|
|
3376
|
+
|
|
3377
|
+
for propname, propvalu in sode.get('props', {}).items():
|
|
3378
|
+
edits.append(
|
|
3379
|
+
(EDIT_PROP_DEL, (propname, *propvalu), ())
|
|
3380
|
+
)
|
|
3381
|
+
|
|
3382
|
+
for tagname, tprops in sode.get('tagprops', {}).items():
|
|
3383
|
+
for propname, propvalu in tprops.items():
|
|
3384
|
+
edits.append(
|
|
3385
|
+
(EDIT_TAGPROP_DEL, (tagname, propname, *propvalu), ())
|
|
3386
|
+
)
|
|
3387
|
+
|
|
3388
|
+
for tagname, tagvalu in sode.get('tags', {}).items():
|
|
3389
|
+
edits.append(
|
|
3390
|
+
(EDIT_TAG_DEL, (tagname, tagvalu), ())
|
|
3391
|
+
)
|
|
3392
|
+
|
|
3393
|
+
# EDIT_NODE_DEL will delete all nodedata and n1 edges if there is a valu in the sode
|
|
3394
|
+
if (valu := sode.get('valu')):
|
|
3395
|
+
edits.append(
|
|
3396
|
+
(EDIT_NODE_DEL, valu, ())
|
|
3397
|
+
)
|
|
3398
|
+
else:
|
|
3399
|
+
async for item in self.iterNodeData(buid):
|
|
3400
|
+
edits.append(
|
|
3401
|
+
(EDIT_NODEDATA_DEL, item, ())
|
|
3402
|
+
)
|
|
3403
|
+
await asyncio.sleep(0)
|
|
3404
|
+
|
|
3405
|
+
async for edge in self.iterNodeEdgesN1(buid):
|
|
3406
|
+
edits.append(
|
|
3407
|
+
(EDIT_EDGE_DEL, edge, ())
|
|
3408
|
+
)
|
|
3409
|
+
await asyncio.sleep(0)
|
|
3410
|
+
|
|
3411
|
+
nodeedits.append((buid, formname, edits))
|
|
3412
|
+
|
|
3413
|
+
n2edges = {}
|
|
3414
|
+
n1iden = s_common.ehex(buid)
|
|
3415
|
+
async for verb, n2iden in self.iterNodeEdgesN2(buid):
|
|
3416
|
+
n2edges.setdefault(n2iden, []).append((verb, n1iden))
|
|
3417
|
+
await asyncio.sleep(0)
|
|
3418
|
+
|
|
3419
|
+
n2forms = {}
|
|
3420
|
+
@s_cache.memoize()
|
|
3421
|
+
def getN2Form(n2iden):
|
|
3422
|
+
buid = s_common.uhex(n2iden)
|
|
3423
|
+
if (form := n2forms.get(buid)) is not None: # pragma: no cover
|
|
3424
|
+
return form
|
|
3425
|
+
|
|
3426
|
+
n2sode = self._getStorNode(buid)
|
|
3427
|
+
form = n2sode.get('form')
|
|
3428
|
+
n2forms[buid] = form
|
|
3429
|
+
return form
|
|
3430
|
+
|
|
3431
|
+
changed = False
|
|
3432
|
+
|
|
3433
|
+
async def batchEdits(size=1000):
|
|
3434
|
+
if len(nodeedits) < size:
|
|
3435
|
+
return changed
|
|
3436
|
+
|
|
3437
|
+
_, changes = await self.saveNodeEdits(nodeedits, meta)
|
|
3438
|
+
|
|
3439
|
+
nodeedits.clear()
|
|
3440
|
+
|
|
3441
|
+
if changed: # pragma: no cover
|
|
3442
|
+
return changed
|
|
3443
|
+
|
|
3444
|
+
return any(c[2] for c in changes)
|
|
3445
|
+
|
|
3446
|
+
for n2iden, edges in n2edges.items():
|
|
3447
|
+
edits = [(EDIT_EDGE_DEL, edge, ()) for edge in edges]
|
|
3448
|
+
nodeedits.append((s_common.uhex(n2iden), getN2Form(n2iden), edits))
|
|
3449
|
+
|
|
3450
|
+
changed = await batchEdits()
|
|
3451
|
+
|
|
3452
|
+
return await batchEdits(size=1)
|
|
3354
3453
|
|
|
3355
3454
|
async def delStorNodeProp(self, buid, prop, meta):
|
|
3356
3455
|
pprop = self.core.model.reqProp(prop)
|
|
@@ -3363,7 +3462,49 @@ class Layer(s_nexus.Pusher):
|
|
|
3363
3462
|
nodeedits = [(buid, oldp_formname, [del_edit])]
|
|
3364
3463
|
|
|
3365
3464
|
_, changes = await self.saveNodeEdits(nodeedits, meta)
|
|
3366
|
-
return
|
|
3465
|
+
return any(c[2] for c in changes)
|
|
3466
|
+
|
|
3467
|
+
async def delNodeData(self, buid, meta, name=None):
|
|
3468
|
+
'''
|
|
3469
|
+
Delete nodedata from a node in this layer. If name is not specified, delete all nodedata.
|
|
3470
|
+
'''
|
|
3471
|
+
sode = self._getStorNode(buid)
|
|
3472
|
+
if sode is None: # pragma: no cover
|
|
3473
|
+
return False
|
|
3474
|
+
|
|
3475
|
+
edits = []
|
|
3476
|
+
if name is None:
|
|
3477
|
+
async for item in self.iterNodeData(buid):
|
|
3478
|
+
edits.append((EDIT_NODEDATA_DEL, item, ()))
|
|
3479
|
+
await asyncio.sleep(0)
|
|
3480
|
+
|
|
3481
|
+
elif await self.hasNodeData(buid, name):
|
|
3482
|
+
edits.append((EDIT_NODEDATA_DEL, (name, None), ()))
|
|
3483
|
+
|
|
3484
|
+
if not edits:
|
|
3485
|
+
return False
|
|
3486
|
+
|
|
3487
|
+
nodeedits = [(buid, sode.get('form'), edits)]
|
|
3488
|
+
|
|
3489
|
+
_, changes = await self.saveNodeEdits(nodeedits, meta)
|
|
3490
|
+
return any(c[2] for c in changes)
|
|
3491
|
+
|
|
3492
|
+
async def delEdge(self, n1buid, verb, n2buid, meta):
|
|
3493
|
+
sode = self._getStorNode(n1buid)
|
|
3494
|
+
if sode is None: # pragma: no cover
|
|
3495
|
+
return False
|
|
3496
|
+
|
|
3497
|
+
if not await self.hasNodeEdge(n1buid, verb, n2buid): # pragma: no cover
|
|
3498
|
+
return False
|
|
3499
|
+
|
|
3500
|
+
edits = [
|
|
3501
|
+
(EDIT_EDGE_DEL, (verb, s_common.ehex(n2buid)), ())
|
|
3502
|
+
]
|
|
3503
|
+
|
|
3504
|
+
nodeedits = [(n1buid, sode.get('form'), edits)]
|
|
3505
|
+
|
|
3506
|
+
_, changes = await self.saveNodeEdits(nodeedits, meta)
|
|
3507
|
+
return any(c[2] for c in changes)
|
|
3367
3508
|
|
|
3368
3509
|
async def storNodeEdits(self, nodeedits, meta):
|
|
3369
3510
|
|
synapse/lib/parser.py
CHANGED
|
@@ -669,6 +669,7 @@ ruleClassMap = {
|
|
|
669
669
|
'condsetoper': s_ast.CondSetOper,
|
|
670
670
|
'condtrysetoper': lambda astinfo, kids: s_ast.CondSetOper(astinfo, kids, errok=True),
|
|
671
671
|
'condsubq': s_ast.SubqCond,
|
|
672
|
+
'derefprops': s_ast.DerefProps,
|
|
672
673
|
'dollarexpr': s_ast.DollarExpr,
|
|
673
674
|
'edgeaddn1': s_ast.EditEdgeAdd,
|
|
674
675
|
'edgedeln1': s_ast.EditEdgeDel,
|
synapse/lib/rstorm.py
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import sys
|
|
2
3
|
import copy
|
|
4
|
+
import shlex
|
|
3
5
|
import pprint
|
|
4
6
|
import logging
|
|
7
|
+
import argparse
|
|
5
8
|
import contextlib
|
|
9
|
+
import subprocess
|
|
6
10
|
import collections
|
|
7
11
|
|
|
8
12
|
import vcr
|
|
@@ -25,7 +29,7 @@ import synapse.tools.storm as s_storm
|
|
|
25
29
|
import synapse.tools.genpkg as s_genpkg
|
|
26
30
|
|
|
27
31
|
|
|
28
|
-
re_directive = regex.compile(r'^\.\.\s(storm.*|[^:])::(?:\s(.*)$|$)')
|
|
32
|
+
re_directive = regex.compile(r'^\.\.\s(shell.*|storm.*|[^:])::(?:\s(.*)$|$)')
|
|
29
33
|
|
|
30
34
|
logger = logging.getLogger(__name__)
|
|
31
35
|
|
|
@@ -48,7 +52,6 @@ class OutPutRst(s_output.OutPutStr):
|
|
|
48
52
|
|
|
49
53
|
return s_output.OutPutStr.printf(self, mesg, addnl)
|
|
50
54
|
|
|
51
|
-
|
|
52
55
|
class StormOutput(s_cmds_cortex.StormCmd):
|
|
53
56
|
'''
|
|
54
57
|
Produce standard output from a stream of storm runtime messages.
|
|
@@ -329,6 +332,8 @@ class StormRst(s_base.Base):
|
|
|
329
332
|
self.core = None
|
|
330
333
|
|
|
331
334
|
self.handlers = {
|
|
335
|
+
'shell': self._handleShell,
|
|
336
|
+
'shell-env': self._handleShellEnv,
|
|
332
337
|
'storm': self._handleStorm,
|
|
333
338
|
'storm-cli': self._handleStormCli,
|
|
334
339
|
'storm-pkg': self._handleStormPkg,
|
|
@@ -339,6 +344,7 @@ class StormRst(s_base.Base):
|
|
|
339
344
|
'storm-cortex': self._handleStormCortex,
|
|
340
345
|
'storm-envvar': self._handleStormEnvVar,
|
|
341
346
|
'storm-expect': self._handleStormExpect,
|
|
347
|
+
'storm-python-path': self._handlePythonPath,
|
|
342
348
|
'storm-multiline': self._handleStormMultiline,
|
|
343
349
|
'storm-mock-http': self._handleStormMockHttp,
|
|
344
350
|
'storm-vcr-opts': self._handleStormVcrOpts,
|
|
@@ -369,6 +375,22 @@ class StormRst(s_base.Base):
|
|
|
369
375
|
raise s_exc.NoSuchName(mesg=f'The {directive} directive is not supported', directive=directive)
|
|
370
376
|
return handler
|
|
371
377
|
|
|
378
|
+
async def _handlePythonPath(self, text):
|
|
379
|
+
'''
|
|
380
|
+
Add the text to sys.path.
|
|
381
|
+
|
|
382
|
+
Args:
|
|
383
|
+
text: The path to add.
|
|
384
|
+
'''
|
|
385
|
+
if not os.path.isdir(text):
|
|
386
|
+
raise s_exc.NoSuchDir(mesg=f'The path {text} is not a directory', path=text)
|
|
387
|
+
if text not in sys.path:
|
|
388
|
+
logger.debug(f'Inserting {text} into sys.path')
|
|
389
|
+
sys.path.insert(0, text)
|
|
390
|
+
def onfini():
|
|
391
|
+
sys.path.remove(text)
|
|
392
|
+
self.onfini(onfini)
|
|
393
|
+
|
|
372
394
|
async def _handleStorm(self, text):
|
|
373
395
|
'''
|
|
374
396
|
Run a Storm command and generate text from the output.
|
|
@@ -596,6 +618,65 @@ class StormRst(s_base.Base):
|
|
|
596
618
|
raise s_exc.NoSuchCtor(mesg=f'Failed to get callback "{text}"', ctor=text)
|
|
597
619
|
self.context['storm-vcr-callback'] = cb
|
|
598
620
|
|
|
621
|
+
async def _handleShell(self, text):
|
|
622
|
+
'''
|
|
623
|
+
Execute shell with the supplied arguments.
|
|
624
|
+
'''
|
|
625
|
+
parser = argparse.ArgumentParser(add_help=False)
|
|
626
|
+
parser.add_argument('--include-stderr', action='store_true', help='Include stderr in output.')
|
|
627
|
+
parser.add_argument('--hide-query', action='store_true', help='Do not include the command in the output.')
|
|
628
|
+
parser.add_argument('--fail-ok', action='store_true', help='Non-zero return values are non-fatal.')
|
|
629
|
+
opts, args = parser.parse_known_args(shlex.split(text))
|
|
630
|
+
|
|
631
|
+
# Remove any command line arguments
|
|
632
|
+
query = text
|
|
633
|
+
query = query.replace('--include-stderr', '')
|
|
634
|
+
query = query.replace('--hide-query', '')
|
|
635
|
+
query = query.replace('--fail-ok', '')
|
|
636
|
+
query = query.strip()
|
|
637
|
+
|
|
638
|
+
env = dict(os.environ)
|
|
639
|
+
env.update(self.context.get('shell-env', {}))
|
|
640
|
+
|
|
641
|
+
stderr = None
|
|
642
|
+
if opts.include_stderr:
|
|
643
|
+
stderr = subprocess.STDOUT
|
|
644
|
+
|
|
645
|
+
proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=stderr, env=env, text=True)
|
|
646
|
+
if proc.returncode != 0 and not opts.fail_ok:
|
|
647
|
+
mesg = f'Error when executing shell directive: {text} (rv: {proc.returncode})'
|
|
648
|
+
raise s_exc.SynErr(mesg=mesg)
|
|
649
|
+
|
|
650
|
+
self._printf('::\n\n')
|
|
651
|
+
|
|
652
|
+
if not opts.hide_query:
|
|
653
|
+
self._printf(f' {query}\n\n')
|
|
654
|
+
|
|
655
|
+
for line in proc.stdout.splitlines():
|
|
656
|
+
self._printf(f' {line}\n')
|
|
657
|
+
|
|
658
|
+
self._printf('\n\n')
|
|
659
|
+
|
|
660
|
+
async def _handleShellEnv(self, text):
|
|
661
|
+
'''
|
|
662
|
+
Env to use in subsequent shell queries.
|
|
663
|
+
|
|
664
|
+
Args:
|
|
665
|
+
text (str): [KEY=VALUE ...]
|
|
666
|
+
Note: No arguments will reset the shell environment.
|
|
667
|
+
'''
|
|
668
|
+
text = text.strip()
|
|
669
|
+
if not text:
|
|
670
|
+
return self.context.pop('shell-env')
|
|
671
|
+
|
|
672
|
+
env = {}
|
|
673
|
+
|
|
674
|
+
for item in text.split(' '):
|
|
675
|
+
key, val = item.split('=')
|
|
676
|
+
env[key] = val
|
|
677
|
+
|
|
678
|
+
self.context['shell-env'] = env
|
|
679
|
+
|
|
599
680
|
async def _readline(self, line):
|
|
600
681
|
|
|
601
682
|
match = re_directive.match(line)
|
synapse/lib/schemas.py
CHANGED
|
@@ -816,6 +816,10 @@ _reqValidPkgdefSchema = {
|
|
|
816
816
|
'items': {'type': 'array',
|
|
817
817
|
'items': {'type': 'string'}},
|
|
818
818
|
},
|
|
819
|
+
'asroot:ondeny:import': {
|
|
820
|
+
'type': 'string',
|
|
821
|
+
'enum': ['allow', 'warn', 'deny'],
|
|
822
|
+
},
|
|
819
823
|
},
|
|
820
824
|
'additionalProperties': True,
|
|
821
825
|
'required': ['name', 'storm']
|
synapse/lib/snap.py
CHANGED
|
@@ -1123,22 +1123,30 @@ class Snap(s_base.Base):
|
|
|
1123
1123
|
mesg = f'No property named "{full}".'
|
|
1124
1124
|
raise s_exc.NoSuchProp(mesg=mesg)
|
|
1125
1125
|
|
|
1126
|
-
if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
return
|
|
1126
|
+
if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr in ('=', '?='):
|
|
1127
|
+
excignore = ()
|
|
1128
|
+
if cmpr == '?=':
|
|
1129
|
+
excignore = (s_exc.BadTypeValu,)
|
|
1131
1130
|
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1131
|
+
try:
|
|
1132
|
+
if prop.isform:
|
|
1133
|
+
if (node := await self._getGuidNodeByDict(prop, valu)) is not None:
|
|
1134
|
+
yield node
|
|
1135
|
+
return
|
|
1136
1136
|
|
|
1137
|
-
|
|
1138
|
-
|
|
1137
|
+
fname = prop.type.name
|
|
1138
|
+
if (form := prop.modl.form(fname)) is None:
|
|
1139
|
+
mesg = f'The property "{full}" type "{fname}" is not a form and cannot be lifted using a dictionary.'
|
|
1140
|
+
raise s_exc.BadTypeValu(mesg=mesg)
|
|
1139
1141
|
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
+
if (node := await self._getGuidNodeByDict(form, valu)) is None:
|
|
1143
|
+
return
|
|
1144
|
+
|
|
1145
|
+
norm = False
|
|
1146
|
+
valu = node.ndef[1]
|
|
1147
|
+
|
|
1148
|
+
except excignore:
|
|
1149
|
+
return
|
|
1142
1150
|
|
|
1143
1151
|
if norm:
|
|
1144
1152
|
cmprvals = prop.type.getStorCmprs(cmpr, valu)
|
synapse/lib/stormhttp.py
CHANGED
|
@@ -90,7 +90,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
90
90
|
For APIs that accept an ssl_opts argument, the dictionary may contain the following values::
|
|
91
91
|
|
|
92
92
|
({
|
|
93
|
-
'verify': <
|
|
93
|
+
'verify': <boolean> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
|
|
94
94
|
'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
|
|
95
95
|
'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
|
|
96
96
|
'ca_cert': <str> - A PEM encoded full chain CA certificate for use when verifying the request.
|
|
@@ -116,9 +116,9 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
116
116
|
'default': None},
|
|
117
117
|
{'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
|
|
118
118
|
'default': 300},
|
|
119
|
-
{'name': 'allow_redirects', 'type': '
|
|
119
|
+
{'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to false, do not follow redirects.',
|
|
120
120
|
'default': True},
|
|
121
|
-
{'name': 'proxy', 'type': ['
|
|
121
|
+
{'name': 'proxy', 'type': ['boolean', 'str'],
|
|
122
122
|
'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
|
|
123
123
|
{'name': 'ssl_opts', 'type': 'dict',
|
|
124
124
|
'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
|
|
@@ -141,7 +141,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
141
141
|
'default': None},
|
|
142
142
|
{'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
|
|
143
143
|
'default': 300},
|
|
144
|
-
{'name': 'allow_redirects', 'type': '
|
|
144
|
+
{'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to false, do not follow redirects.',
|
|
145
145
|
'default': True},
|
|
146
146
|
{'name': 'fields', 'type': 'list',
|
|
147
147
|
'desc': 'A list of info dictionaries containing the name, value or sha256, '
|
|
@@ -150,7 +150,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
150
150
|
'and the corresponding file will be uploaded as the value for '
|
|
151
151
|
'the field.',
|
|
152
152
|
'default': None},
|
|
153
|
-
{'name': 'proxy', 'type': ['
|
|
153
|
+
{'name': 'proxy', 'type': ['boolean', 'str'],
|
|
154
154
|
'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
|
|
155
155
|
{'name': 'ssl_opts', 'type': 'dict',
|
|
156
156
|
'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
|
|
@@ -170,9 +170,9 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
170
170
|
'default': None},
|
|
171
171
|
{'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
|
|
172
172
|
'default': 300, },
|
|
173
|
-
{'name': 'allow_redirects', 'type': '
|
|
173
|
+
{'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to true, follow redirects.',
|
|
174
174
|
'default': False},
|
|
175
|
-
{'name': 'proxy', 'type': ['
|
|
175
|
+
{'name': 'proxy', 'type': ['boolean', 'str'],
|
|
176
176
|
'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
|
|
177
177
|
{'name': 'ssl_opts', 'type': 'dict',
|
|
178
178
|
'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
|
|
@@ -196,7 +196,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
196
196
|
'default': None},
|
|
197
197
|
{'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
|
|
198
198
|
'default': 300},
|
|
199
|
-
{'name': 'allow_redirects', 'type': '
|
|
199
|
+
{'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to false, do not follow redirects.',
|
|
200
200
|
'default': True},
|
|
201
201
|
{'name': 'fields', 'type': 'list',
|
|
202
202
|
'desc': 'A list of info dictionaries containing the name, value or sha256, '
|
|
@@ -205,7 +205,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
205
205
|
'and the corresponding file will be uploaded as the value for '
|
|
206
206
|
'the field.',
|
|
207
207
|
'default': None},
|
|
208
|
-
{'name': 'proxy', 'type': ['
|
|
208
|
+
{'name': 'proxy', 'type': ['boolean', 'str'],
|
|
209
209
|
'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
|
|
210
210
|
{'name': 'ssl_opts', 'type': 'dict',
|
|
211
211
|
'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
|
|
@@ -226,7 +226,7 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
226
226
|
'default': 300},
|
|
227
227
|
{'name': 'params', 'type': 'dict', 'desc': 'Optional parameters which may be passed to the connection request.',
|
|
228
228
|
'default': None},
|
|
229
|
-
{'name': 'proxy', 'type': ['
|
|
229
|
+
{'name': 'proxy', 'type': ['boolean', 'str'],
|
|
230
230
|
'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
|
|
231
231
|
{'name': 'ssl_opts', 'type': 'dict',
|
|
232
232
|
'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
|