synapse 2.220.0__py311-none-any.whl → 2.222.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (52) hide show
  1. synapse/cortex.py +34 -14
  2. synapse/data/lark/storm.lark +9 -6
  3. synapse/lib/ast.py +8 -2
  4. synapse/lib/layer.py +149 -8
  5. synapse/lib/parser.py +1 -0
  6. synapse/lib/rstorm.py +83 -2
  7. synapse/lib/schemas.py +4 -0
  8. synapse/lib/snap.py +21 -13
  9. synapse/lib/stormhttp.py +10 -10
  10. synapse/lib/stormlib/aha.py +3 -3
  11. synapse/lib/stormlib/auth.py +11 -11
  12. synapse/lib/stormlib/cell.py +1 -1
  13. synapse/lib/stormlib/cortex.py +10 -10
  14. synapse/lib/stormlib/env.py +4 -5
  15. synapse/lib/stormlib/ethereum.py +1 -1
  16. synapse/lib/stormlib/gen.py +3 -3
  17. synapse/lib/stormlib/hex.py +2 -2
  18. synapse/lib/stormlib/imap.py +2 -2
  19. synapse/lib/stormlib/infosec.py +2 -2
  20. synapse/lib/stormlib/iters.py +2 -2
  21. synapse/lib/stormlib/model.py +5 -5
  22. synapse/lib/stormlib/notifications.py +1 -1
  23. synapse/lib/stormlib/oauth.py +2 -2
  24. synapse/lib/stormlib/project.py +3 -3
  25. synapse/lib/stormlib/scrape.py +2 -1
  26. synapse/lib/stormlib/smtp.py +3 -3
  27. synapse/lib/stormlib/stats.py +2 -2
  28. synapse/lib/stormlib/stix.py +2 -2
  29. synapse/lib/stormlib/utils.py +19 -0
  30. synapse/lib/stormlib/vault.py +1 -1
  31. synapse/lib/stormlib/xml.py +2 -2
  32. synapse/lib/stormlib/yaml.py +1 -1
  33. synapse/lib/stormtypes.py +182 -64
  34. synapse/lib/version.py +2 -2
  35. synapse/models/orgs.py +3 -0
  36. synapse/tests/test_lib_grammar.py +4 -4
  37. synapse/tests/test_lib_layer.py +86 -67
  38. synapse/tests/test_lib_rstorm.py +180 -0
  39. synapse/tests/test_lib_storm.py +80 -1
  40. synapse/tests/test_lib_stormlib_auth.py +84 -0
  41. synapse/tests/test_lib_stormlib_cortex.py +1 -0
  42. synapse/tests/test_lib_stormlib_env.py +3 -1
  43. synapse/tests/test_lib_stormlib_utils.py +10 -0
  44. synapse/tests/test_lib_stormtypes.py +576 -2
  45. synapse/tests/test_model_orgs.py +6 -1
  46. synapse/tools/aha/list.py +9 -9
  47. synapse/tools/aha/provision/service.py +2 -2
  48. {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/METADATA +1 -1
  49. {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/RECORD +52 -52
  50. {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/WHEEL +0 -0
  51. {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/licenses/LICENSE +0 -0
  52. {synapse-2.220.0.dist-info → synapse-2.222.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py CHANGED
@@ -624,7 +624,8 @@ class CoreApi(s_cell.CellApi):
624
624
  Returns:
625
625
  AsyncIterator[Tuple(buid, valu)]
626
626
  '''
627
- self.user.confirm(('layer', 'lift', layriden))
627
+ if not self.user.allowed(('layer', 'lift', layriden)):
628
+ self.user.confirm(('layer', 'read', layriden))
628
629
  async for item in self.cell.iterFormRows(layriden, form, stortype=stortype, startvalu=startvalu):
629
630
  yield item
630
631
 
@@ -642,7 +643,8 @@ class CoreApi(s_cell.CellApi):
642
643
  Returns:
643
644
  AsyncIterator[Tuple(buid, valu)]
644
645
  '''
645
- self.user.confirm(('layer', 'lift', layriden))
646
+ if not self.user.allowed(('layer', 'lift', layriden)):
647
+ self.user.confirm(('layer', 'read', layriden))
646
648
  async for item in self.cell.iterPropRows(layriden, form, prop, stortype=stortype, startvalu=startvalu):
647
649
  yield item
648
650
 
@@ -659,7 +661,8 @@ class CoreApi(s_cell.CellApi):
659
661
  Returns:
660
662
  AsyncIterator[Tuple(buid, valu)]
661
663
  '''
662
- self.user.confirm(('layer', 'lift', layriden))
664
+ if not self.user.allowed(('layer', 'lift', layriden)):
665
+ self.user.confirm(('layer', 'read', layriden))
663
666
  async for item in self.cell.iterUnivRows(layriden, prop, stortype=stortype, startvalu=startvalu):
664
667
  yield item
665
668
 
@@ -680,7 +683,8 @@ class CoreApi(s_cell.CellApi):
680
683
  This yields (buid, (tagvalu, form)) instead of just buid, valu in order to allow resuming an interrupted
681
684
  call by feeding the last value retrieved into starttupl
682
685
  '''
683
- self.user.confirm(('layer', 'lift', layriden))
686
+ if not self.user.allowed(('layer', 'lift', layriden)):
687
+ self.user.confirm(('layer', 'read', layriden))
684
688
  async for item in self.cell.iterTagRows(layriden, tag, form=form, starttupl=starttupl):
685
689
  yield item
686
690
 
@@ -699,7 +703,8 @@ class CoreApi(s_cell.CellApi):
699
703
  Returns:
700
704
  AsyncIterator[Tuple(buid, valu)]
701
705
  '''
702
- self.user.confirm(('layer', 'lift', layriden))
706
+ if not self.user.allowed(('layer', 'lift', layriden)):
707
+ self.user.confirm(('layer', 'read', layriden))
703
708
  async for item in self.cell.iterTagPropRows(layriden, tag, prop, form=form, stortype=stortype,
704
709
  startvalu=startvalu):
705
710
  yield item
@@ -1366,6 +1371,30 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1366
1371
 
1367
1372
  def _initCorePerms(self):
1368
1373
  self._cortex_permdefs.extend((
1374
+ {'perm': ('axon', 'upload'), 'gate': 'cortex',
1375
+ 'desc': 'Controls the ability to upload a file to the Axon.'},
1376
+ {'perm': ('axon', 'get'), 'gate': 'cortex',
1377
+ 'desc': 'Controls the ability to retrieve a file from the Axon.'},
1378
+ {'perm': ('axon', 'has'), 'gate': 'cortex',
1379
+ 'desc': 'Controls the ability to check if the Axon contains a file.'},
1380
+ {'perm': ('axon', 'del'), 'gate': 'cortex',
1381
+ 'desc': 'Controls the ability to remove a file from the Axon.'},
1382
+
1383
+ {'perm': ('layer', 'add'), 'gate': 'cortex',
1384
+ 'desc': 'Controls the ability to add Layers to the cortex.'},
1385
+ {'perm': ('layer', 'del'), 'gate': 'cortex',
1386
+ 'desc': 'Controls the ability to remove Layers from the cortex.'},
1387
+ {'perm': ('layer', 'read'), 'gate': 'layer',
1388
+ 'desc': 'Controls the ability to read/lift from a Layer.'},
1389
+ {'perm': ('layer', 'read', '<iden>'), 'gate': 'cortex',
1390
+ 'desc': 'Controls the ability to read/lift from a specific Layer.'},
1391
+ {'perm': ('layer', 'set', '<name>'), 'gate': 'layer',
1392
+ 'desc': 'Controls the ability to configure properties of a Layer.'},
1393
+ {'perm': ('layer', 'write'), 'gate': 'layer',
1394
+ 'desc': 'Controls the ability to write to a Layer.'},
1395
+ {'perm': ('layer', 'write', '<iden>'), 'gate': 'cortex',
1396
+ 'desc': 'Controls the ability to write to a specific Layer.'},
1397
+
1369
1398
  {'perm': ('model', 'form', 'add'), 'gate': 'cortex',
1370
1399
  'desc': 'Controls access to adding extended model forms.'},
1371
1400
  {'perm': ('model', 'form', 'add', '<form>'), 'gate': 'cortex',
@@ -1519,15 +1548,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1519
1548
  {'perm': ('view', 'set', '<setting>'), 'gate': 'view',
1520
1549
  'desc': 'Controls access to change view settings.',
1521
1550
  'ex': 'view.set.name'},
1522
-
1523
- {'perm': ('axon', 'upload'), 'gate': 'cortex',
1524
- 'desc': 'Controls the ability to upload a file to the Axon.'},
1525
- {'perm': ('axon', 'get'), 'gate': 'cortex',
1526
- 'desc': 'Controls the ability to retrieve a file from the Axon.'},
1527
- {'perm': ('axon', 'has'), 'gate': 'cortex',
1528
- 'desc': 'Controls the ability to check if the Axon contains a file.'},
1529
- {'perm': ('axon', 'del'), 'gate': 'cortex',
1530
- 'desc': 'Controls the ability to remove a file from the Axon.'},
1531
1551
  ))
1532
1552
  for pdef in self._cortex_permdefs:
1533
1553
  s_schemas.reqValidPermDef(pdef)
@@ -238,10 +238,10 @@ liftreverse: _REVERSE "(" (liftformtag | liftpropby | liftprop | liftbyarray | l
238
238
 
239
239
  _DEREF.3: /\*(?=\$)/
240
240
 
241
- liftformtag: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname [_cmpr _valu]
242
- liftpropby: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) _cmpr _valu
243
- liftprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu)
244
- liftbyarray: ((PROPS | EMBEDPROPS | UNIVNAME) | _DEREF _varvalu) "*[" _safe_cmpr _valu "]"
241
+ liftformtag: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname [_cmpr _valu]
242
+ liftpropby: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) _cmpr _valu
243
+ liftprop: (PROPS | UNIVNAME | WILDPROPS | derefprops)
244
+ liftbyarray: (PROPS | EMBEDPROPS | UNIVNAME | derefprops) "*[" _safe_cmpr _valu "]"
245
245
  lifttagtag: (_HASH | _HASHSPACE) tagname [_cmpr _valu]
246
246
  liftbytag: (tagname | tagnamewithspace) [_cmpr _valu]
247
247
  liftbytagprop: (tagprop | tagpropwithspace) [_cmpr _valu]
@@ -249,7 +249,7 @@ liftbyformtagprop: formtagprop [_cmpr _valu]
249
249
 
250
250
  tagprop: tagname _COLONNOSPACE (BASEPROP | _varvalu)
251
251
  tagpropwithspace: tagnamewithspace _COLONNOSPACE (BASEPROP | _varvalu) -> tagprop
252
- formtagprop: ((PROPS | UNIVNAME | WILDPROPS) | _DEREF _varvalu) tagname _COLONNOSPACE (BASEPROP | _varvalu)
252
+ formtagprop: (PROPS | UNIVNAME | WILDPROPS | derefprops) tagname _COLONNOSPACE (BASEPROP | _varvalu)
253
253
  _COLONNOSPACE.2: /(?<!\s):/
254
254
 
255
255
  _funcarg: (VARTOKN [EQNOSPACE _valu])
@@ -484,7 +484,7 @@ EMBEDPROPS.2: /[a-z_][a-z0-9_]*(:[a-z0-9_]+)+((\:\:|\:|\.)[a-z0-9_]+)*(?![:.a-z0
484
484
  UNIVNAME.2: /(?<=^|[\s\|\{\(\[+=-])\.[a-z_][a-z0-9_]*([:.][a-z0-9_]+)*/
485
485
  univprop: UNIVNAME | "." _varvalu
486
486
  // A full property or a universal property
487
- formname: PROPS | _DEREF _varvalu
487
+ formname: PROPS | derefprops
488
488
  // A relative property
489
489
  relprop: RELNAME | _COLONDOLLAR _varvaluatom
490
490
 
@@ -494,6 +494,9 @@ RELNAME: /(?<!\w):\.?[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
494
494
  // Similar to PROPS but does not require a colon
495
495
  BASEPROP: /[a-z_][a-z0-9_]*(?:(\:\:|\:|\.)[a-z_][a-z0-9_]*)*/
496
496
 
497
+ // Derefed variable for prop names
498
+ derefprops: _DEREF _varvalu
499
+
497
500
  // The entry point for a $(...) expression. The initial dollar sign is now optional
498
501
  _dollarexprs: "$"? dollaroper
499
502
 
synapse/lib/ast.py CHANGED
@@ -2,7 +2,6 @@ import types
2
2
  import asyncio
3
3
  import decimal
4
4
  import fnmatch
5
- import hashlib
6
5
  import logging
7
6
  import binascii
8
7
  import itertools
@@ -1833,7 +1832,7 @@ class LiftProp(LiftOper):
1833
1832
 
1834
1833
  assert len(self.kids) == 1
1835
1834
 
1836
- name = await tostr(await self.kids[0].compute(runt, path))
1835
+ name = await self.kids[0].compute(runt, path)
1837
1836
 
1838
1837
  prop = runt.model.props.get(name)
1839
1838
  if prop is not None:
@@ -4029,6 +4028,13 @@ class FormName(Value):
4029
4028
  async def compute(self, runt, path):
4030
4029
  return await self.kids[0].compute(runt, path)
4031
4030
 
4031
+ class DerefProps(Value):
4032
+ async def compute(self, runt, path):
4033
+ valu = await toprim(await self.kids[0].compute(runt, path))
4034
+ if (exc := await s_stormtypes.typeerr(valu, str)) is not None:
4035
+ raise self.kids[0].addExcInfo(exc)
4036
+ return valu
4037
+
4032
4038
  class RelProp(PropName):
4033
4039
  pass
4034
4040
 
synapse/lib/layer.py CHANGED
@@ -119,6 +119,7 @@ class LayerApi(s_cell.CellApi):
119
119
 
120
120
  self.layr = layr
121
121
  self.liftperm = ('layer', 'lift', self.layr.iden)
122
+ self.readperm = ('layer', 'read', self.layr.iden)
122
123
  self.writeperm = ('layer', 'write', self.layr.iden)
123
124
 
124
125
  async def iterLayerNodeEdits(self):
@@ -126,7 +127,8 @@ class LayerApi(s_cell.CellApi):
126
127
  Scan the full layer and yield artificial nodeedit sets.
127
128
  '''
128
129
 
129
- await self._reqUserAllowed(self.liftperm)
130
+ if not self.allowed(self.liftperm):
131
+ await self._reqUserAllowed(self.readperm)
130
132
  async for item in self.layr.iterLayerNodeEdits():
131
133
  yield item
132
134
  await asyncio.sleep(0)
@@ -165,13 +167,15 @@ class LayerApi(s_cell.CellApi):
165
167
 
166
168
  Once caught up with storage, yield them in realtime.
167
169
  '''
168
- await self._reqUserAllowed(self.liftperm)
170
+ if not self.allowed(self.liftperm):
171
+ await self._reqUserAllowed(self.readperm)
169
172
  async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
170
173
  yield item
171
174
  await asyncio.sleep(0)
172
175
 
173
176
  async def syncNodeEdits2(self, offs, wait=True):
174
- await self._reqUserAllowed(self.liftperm)
177
+ if not self.allowed(self.liftperm):
178
+ await self._reqUserAllowed(self.readperm)
175
179
  async for item in self.layr.syncNodeEdits2(offs, wait=wait):
176
180
  yield item
177
181
  await asyncio.sleep(0)
@@ -180,18 +184,21 @@ class LayerApi(s_cell.CellApi):
180
184
  '''
181
185
  Returns what will be the *next* nodeedit log index.
182
186
  '''
183
- await self._reqUserAllowed(self.liftperm)
187
+ if not self.allowed(self.liftperm):
188
+ await self._reqUserAllowed(self.readperm)
184
189
  return await self.layr.getEditIndx()
185
190
 
186
191
  async def getEditSize(self):
187
192
  '''
188
193
  Return the total number of (edits, meta) pairs in the layer changelog.
189
194
  '''
190
- await self._reqUserAllowed(self.liftperm)
195
+ if not self.allowed(self.liftperm):
196
+ await self._reqUserAllowed(self.readperm)
191
197
  return await self.layr.getEditSize()
192
198
 
193
199
  async def getIden(self):
194
- await self._reqUserAllowed(self.liftperm)
200
+ if not self.allowed(self.liftperm):
201
+ await self._reqUserAllowed(self.readperm)
195
202
  return self.layr.iden
196
203
 
197
204
  BUID_CACHE_SIZE = 10000
@@ -3350,7 +3357,99 @@ class Layer(s_nexus.Pusher):
3350
3357
  nodeedits = [(buid, newp_formname, [set_edit])]
3351
3358
 
3352
3359
  _, changes = await self.saveNodeEdits(nodeedits, meta)
3353
- return bool(changes[0][2])
3360
+ return any(c[2] for c in changes)
3361
+
3362
+ async def delStorNode(self, buid, meta):
3363
+ '''
3364
+ Delete all node information in this layer.
3365
+
3366
+ Deletes props, tagprops, tags, n1edges, n2edges, nodedata, and node valu.
3367
+ '''
3368
+ sode = self._getStorNode(buid)
3369
+ if sode is None:
3370
+ return False
3371
+
3372
+ formname = sode.get('form')
3373
+
3374
+ edits = []
3375
+ nodeedits = []
3376
+
3377
+ for propname, propvalu in sode.get('props', {}).items():
3378
+ edits.append(
3379
+ (EDIT_PROP_DEL, (propname, *propvalu), ())
3380
+ )
3381
+
3382
+ for tagname, tprops in sode.get('tagprops', {}).items():
3383
+ for propname, propvalu in tprops.items():
3384
+ edits.append(
3385
+ (EDIT_TAGPROP_DEL, (tagname, propname, *propvalu), ())
3386
+ )
3387
+
3388
+ for tagname, tagvalu in sode.get('tags', {}).items():
3389
+ edits.append(
3390
+ (EDIT_TAG_DEL, (tagname, tagvalu), ())
3391
+ )
3392
+
3393
+ # EDIT_NODE_DEL will delete all nodedata and n1 edges if there is a valu in the sode
3394
+ if (valu := sode.get('valu')):
3395
+ edits.append(
3396
+ (EDIT_NODE_DEL, valu, ())
3397
+ )
3398
+ else:
3399
+ async for item in self.iterNodeData(buid):
3400
+ edits.append(
3401
+ (EDIT_NODEDATA_DEL, item, ())
3402
+ )
3403
+ await asyncio.sleep(0)
3404
+
3405
+ async for edge in self.iterNodeEdgesN1(buid):
3406
+ edits.append(
3407
+ (EDIT_EDGE_DEL, edge, ())
3408
+ )
3409
+ await asyncio.sleep(0)
3410
+
3411
+ nodeedits.append((buid, formname, edits))
3412
+
3413
+ n2edges = {}
3414
+ n1iden = s_common.ehex(buid)
3415
+ async for verb, n2iden in self.iterNodeEdgesN2(buid):
3416
+ n2edges.setdefault(n2iden, []).append((verb, n1iden))
3417
+ await asyncio.sleep(0)
3418
+
3419
+ n2forms = {}
3420
+ @s_cache.memoize()
3421
+ def getN2Form(n2iden):
3422
+ buid = s_common.uhex(n2iden)
3423
+ if (form := n2forms.get(buid)) is not None: # pragma: no cover
3424
+ return form
3425
+
3426
+ n2sode = self._getStorNode(buid)
3427
+ form = n2sode.get('form')
3428
+ n2forms[buid] = form
3429
+ return form
3430
+
3431
+ changed = False
3432
+
3433
+ async def batchEdits(size=1000):
3434
+ if len(nodeedits) < size:
3435
+ return changed
3436
+
3437
+ _, changes = await self.saveNodeEdits(nodeedits, meta)
3438
+
3439
+ nodeedits.clear()
3440
+
3441
+ if changed: # pragma: no cover
3442
+ return changed
3443
+
3444
+ return any(c[2] for c in changes)
3445
+
3446
+ for n2iden, edges in n2edges.items():
3447
+ edits = [(EDIT_EDGE_DEL, edge, ()) for edge in edges]
3448
+ nodeedits.append((s_common.uhex(n2iden), getN2Form(n2iden), edits))
3449
+
3450
+ changed = await batchEdits()
3451
+
3452
+ return await batchEdits(size=1)
3354
3453
 
3355
3454
  async def delStorNodeProp(self, buid, prop, meta):
3356
3455
  pprop = self.core.model.reqProp(prop)
@@ -3363,7 +3462,49 @@ class Layer(s_nexus.Pusher):
3363
3462
  nodeedits = [(buid, oldp_formname, [del_edit])]
3364
3463
 
3365
3464
  _, changes = await self.saveNodeEdits(nodeedits, meta)
3366
- return bool(changes[0][2])
3465
+ return any(c[2] for c in changes)
3466
+
3467
+ async def delNodeData(self, buid, meta, name=None):
3468
+ '''
3469
+ Delete nodedata from a node in this layer. If name is not specified, delete all nodedata.
3470
+ '''
3471
+ sode = self._getStorNode(buid)
3472
+ if sode is None: # pragma: no cover
3473
+ return False
3474
+
3475
+ edits = []
3476
+ if name is None:
3477
+ async for item in self.iterNodeData(buid):
3478
+ edits.append((EDIT_NODEDATA_DEL, item, ()))
3479
+ await asyncio.sleep(0)
3480
+
3481
+ elif await self.hasNodeData(buid, name):
3482
+ edits.append((EDIT_NODEDATA_DEL, (name, None), ()))
3483
+
3484
+ if not edits:
3485
+ return False
3486
+
3487
+ nodeedits = [(buid, sode.get('form'), edits)]
3488
+
3489
+ _, changes = await self.saveNodeEdits(nodeedits, meta)
3490
+ return any(c[2] for c in changes)
3491
+
3492
+ async def delEdge(self, n1buid, verb, n2buid, meta):
3493
+ sode = self._getStorNode(n1buid)
3494
+ if sode is None: # pragma: no cover
3495
+ return False
3496
+
3497
+ if not await self.hasNodeEdge(n1buid, verb, n2buid): # pragma: no cover
3498
+ return False
3499
+
3500
+ edits = [
3501
+ (EDIT_EDGE_DEL, (verb, s_common.ehex(n2buid)), ())
3502
+ ]
3503
+
3504
+ nodeedits = [(n1buid, sode.get('form'), edits)]
3505
+
3506
+ _, changes = await self.saveNodeEdits(nodeedits, meta)
3507
+ return any(c[2] for c in changes)
3367
3508
 
3368
3509
  async def storNodeEdits(self, nodeedits, meta):
3369
3510
 
synapse/lib/parser.py CHANGED
@@ -669,6 +669,7 @@ ruleClassMap = {
669
669
  'condsetoper': s_ast.CondSetOper,
670
670
  'condtrysetoper': lambda astinfo, kids: s_ast.CondSetOper(astinfo, kids, errok=True),
671
671
  'condsubq': s_ast.SubqCond,
672
+ 'derefprops': s_ast.DerefProps,
672
673
  'dollarexpr': s_ast.DollarExpr,
673
674
  'edgeaddn1': s_ast.EditEdgeAdd,
674
675
  'edgedeln1': s_ast.EditEdgeDel,
synapse/lib/rstorm.py CHANGED
@@ -1,8 +1,12 @@
1
1
  import os
2
+ import sys
2
3
  import copy
4
+ import shlex
3
5
  import pprint
4
6
  import logging
7
+ import argparse
5
8
  import contextlib
9
+ import subprocess
6
10
  import collections
7
11
 
8
12
  import vcr
@@ -25,7 +29,7 @@ import synapse.tools.storm as s_storm
25
29
  import synapse.tools.genpkg as s_genpkg
26
30
 
27
31
 
28
- re_directive = regex.compile(r'^\.\.\s(storm.*|[^:])::(?:\s(.*)$|$)')
32
+ re_directive = regex.compile(r'^\.\.\s(shell.*|storm.*|[^:])::(?:\s(.*)$|$)')
29
33
 
30
34
  logger = logging.getLogger(__name__)
31
35
 
@@ -48,7 +52,6 @@ class OutPutRst(s_output.OutPutStr):
48
52
 
49
53
  return s_output.OutPutStr.printf(self, mesg, addnl)
50
54
 
51
-
52
55
  class StormOutput(s_cmds_cortex.StormCmd):
53
56
  '''
54
57
  Produce standard output from a stream of storm runtime messages.
@@ -329,6 +332,8 @@ class StormRst(s_base.Base):
329
332
  self.core = None
330
333
 
331
334
  self.handlers = {
335
+ 'shell': self._handleShell,
336
+ 'shell-env': self._handleShellEnv,
332
337
  'storm': self._handleStorm,
333
338
  'storm-cli': self._handleStormCli,
334
339
  'storm-pkg': self._handleStormPkg,
@@ -339,6 +344,7 @@ class StormRst(s_base.Base):
339
344
  'storm-cortex': self._handleStormCortex,
340
345
  'storm-envvar': self._handleStormEnvVar,
341
346
  'storm-expect': self._handleStormExpect,
347
+ 'storm-python-path': self._handlePythonPath,
342
348
  'storm-multiline': self._handleStormMultiline,
343
349
  'storm-mock-http': self._handleStormMockHttp,
344
350
  'storm-vcr-opts': self._handleStormVcrOpts,
@@ -369,6 +375,22 @@ class StormRst(s_base.Base):
369
375
  raise s_exc.NoSuchName(mesg=f'The {directive} directive is not supported', directive=directive)
370
376
  return handler
371
377
 
378
+ async def _handlePythonPath(self, text):
379
+ '''
380
+ Add the text to sys.path.
381
+
382
+ Args:
383
+ text: The path to add.
384
+ '''
385
+ if not os.path.isdir(text):
386
+ raise s_exc.NoSuchDir(mesg=f'The path {text} is not a directory', path=text)
387
+ if text not in sys.path:
388
+ logger.debug(f'Inserting {text} into sys.path')
389
+ sys.path.insert(0, text)
390
+ def onfini():
391
+ sys.path.remove(text)
392
+ self.onfini(onfini)
393
+
372
394
  async def _handleStorm(self, text):
373
395
  '''
374
396
  Run a Storm command and generate text from the output.
@@ -596,6 +618,65 @@ class StormRst(s_base.Base):
596
618
  raise s_exc.NoSuchCtor(mesg=f'Failed to get callback "{text}"', ctor=text)
597
619
  self.context['storm-vcr-callback'] = cb
598
620
 
621
+ async def _handleShell(self, text):
622
+ '''
623
+ Execute shell with the supplied arguments.
624
+ '''
625
+ parser = argparse.ArgumentParser(add_help=False)
626
+ parser.add_argument('--include-stderr', action='store_true', help='Include stderr in output.')
627
+ parser.add_argument('--hide-query', action='store_true', help='Do not include the command in the output.')
628
+ parser.add_argument('--fail-ok', action='store_true', help='Non-zero return values are non-fatal.')
629
+ opts, args = parser.parse_known_args(shlex.split(text))
630
+
631
+ # Remove any command line arguments
632
+ query = text
633
+ query = query.replace('--include-stderr', '')
634
+ query = query.replace('--hide-query', '')
635
+ query = query.replace('--fail-ok', '')
636
+ query = query.strip()
637
+
638
+ env = dict(os.environ)
639
+ env.update(self.context.get('shell-env', {}))
640
+
641
+ stderr = None
642
+ if opts.include_stderr:
643
+ stderr = subprocess.STDOUT
644
+
645
+ proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=stderr, env=env, text=True)
646
+ if proc.returncode != 0 and not opts.fail_ok:
647
+ mesg = f'Error when executing shell directive: {text} (rv: {proc.returncode})'
648
+ raise s_exc.SynErr(mesg=mesg)
649
+
650
+ self._printf('::\n\n')
651
+
652
+ if not opts.hide_query:
653
+ self._printf(f' {query}\n\n')
654
+
655
+ for line in proc.stdout.splitlines():
656
+ self._printf(f' {line}\n')
657
+
658
+ self._printf('\n\n')
659
+
660
+ async def _handleShellEnv(self, text):
661
+ '''
662
+ Env to use in subsequent shell queries.
663
+
664
+ Args:
665
+ text (str): [KEY=VALUE ...]
666
+ Note: No arguments will reset the shell environment.
667
+ '''
668
+ text = text.strip()
669
+ if not text:
670
+ return self.context.pop('shell-env')
671
+
672
+ env = {}
673
+
674
+ for item in text.split(' '):
675
+ key, val = item.split('=')
676
+ env[key] = val
677
+
678
+ self.context['shell-env'] = env
679
+
599
680
  async def _readline(self, line):
600
681
 
601
682
  match = re_directive.match(line)
synapse/lib/schemas.py CHANGED
@@ -816,6 +816,10 @@ _reqValidPkgdefSchema = {
816
816
  'items': {'type': 'array',
817
817
  'items': {'type': 'string'}},
818
818
  },
819
+ 'asroot:ondeny:import': {
820
+ 'type': 'string',
821
+ 'enum': ['allow', 'warn', 'deny'],
822
+ },
819
823
  },
820
824
  'additionalProperties': True,
821
825
  'required': ['name', 'storm']
synapse/lib/snap.py CHANGED
@@ -1123,22 +1123,30 @@ class Snap(s_base.Base):
1123
1123
  mesg = f'No property named "{full}".'
1124
1124
  raise s_exc.NoSuchProp(mesg=mesg)
1125
1125
 
1126
- if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr == '=':
1127
- if prop.isform:
1128
- if (node := await self._getGuidNodeByDict(prop, valu)) is not None:
1129
- yield node
1130
- return
1126
+ if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr in ('=', '?='):
1127
+ excignore = ()
1128
+ if cmpr == '?=':
1129
+ excignore = (s_exc.BadTypeValu,)
1131
1130
 
1132
- fname = prop.type.name
1133
- if (form := prop.modl.form(fname)) is None:
1134
- mesg = f'The property "{full}" type "{fname}" is not a form and cannot be lifted using a dictionary.'
1135
- raise s_exc.BadTypeValu(mesg=mesg)
1131
+ try:
1132
+ if prop.isform:
1133
+ if (node := await self._getGuidNodeByDict(prop, valu)) is not None:
1134
+ yield node
1135
+ return
1136
1136
 
1137
- if (node := await self._getGuidNodeByDict(form, valu)) is None:
1138
- return
1137
+ fname = prop.type.name
1138
+ if (form := prop.modl.form(fname)) is None:
1139
+ mesg = f'The property "{full}" type "{fname}" is not a form and cannot be lifted using a dictionary.'
1140
+ raise s_exc.BadTypeValu(mesg=mesg)
1139
1141
 
1140
- norm = False
1141
- valu = node.ndef[1]
1142
+ if (node := await self._getGuidNodeByDict(form, valu)) is None:
1143
+ return
1144
+
1145
+ norm = False
1146
+ valu = node.ndef[1]
1147
+
1148
+ except excignore:
1149
+ return
1142
1150
 
1143
1151
  if norm:
1144
1152
  cmprvals = prop.type.getStorCmprs(cmpr, valu)
synapse/lib/stormhttp.py CHANGED
@@ -90,7 +90,7 @@ class LibHttp(s_stormtypes.Lib):
90
90
  For APIs that accept an ssl_opts argument, the dictionary may contain the following values::
91
91
 
92
92
  ({
93
- 'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
93
+ 'verify': <boolean> - Perform SSL/TLS verification. Is overridden by the ssl_verify argument.
94
94
  'client_cert': <str> - PEM encoded full chain certificate for use in mTLS.
95
95
  'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert.
96
96
  'ca_cert': <str> - A PEM encoded full chain CA certificate for use when verifying the request.
@@ -116,9 +116,9 @@ class LibHttp(s_stormtypes.Lib):
116
116
  'default': None},
117
117
  {'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
118
118
  'default': 300},
119
- {'name': 'allow_redirects', 'type': 'bool', 'desc': 'If set to false, do not follow redirects.',
119
+ {'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to false, do not follow redirects.',
120
120
  'default': True},
121
- {'name': 'proxy', 'type': ['bool', 'str'],
121
+ {'name': 'proxy', 'type': ['boolean', 'str'],
122
122
  'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
123
123
  {'name': 'ssl_opts', 'type': 'dict',
124
124
  'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
@@ -141,7 +141,7 @@ class LibHttp(s_stormtypes.Lib):
141
141
  'default': None},
142
142
  {'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
143
143
  'default': 300},
144
- {'name': 'allow_redirects', 'type': 'bool', 'desc': 'If set to false, do not follow redirects.',
144
+ {'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to false, do not follow redirects.',
145
145
  'default': True},
146
146
  {'name': 'fields', 'type': 'list',
147
147
  'desc': 'A list of info dictionaries containing the name, value or sha256, '
@@ -150,7 +150,7 @@ class LibHttp(s_stormtypes.Lib):
150
150
  'and the corresponding file will be uploaded as the value for '
151
151
  'the field.',
152
152
  'default': None},
153
- {'name': 'proxy', 'type': ['bool', 'str'],
153
+ {'name': 'proxy', 'type': ['boolean', 'str'],
154
154
  'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
155
155
  {'name': 'ssl_opts', 'type': 'dict',
156
156
  'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
@@ -170,9 +170,9 @@ class LibHttp(s_stormtypes.Lib):
170
170
  'default': None},
171
171
  {'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
172
172
  'default': 300, },
173
- {'name': 'allow_redirects', 'type': 'bool', 'desc': 'If set to true, follow redirects.',
173
+ {'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to true, follow redirects.',
174
174
  'default': False},
175
- {'name': 'proxy', 'type': ['bool', 'str'],
175
+ {'name': 'proxy', 'type': ['boolean', 'str'],
176
176
  'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
177
177
  {'name': 'ssl_opts', 'type': 'dict',
178
178
  'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
@@ -196,7 +196,7 @@ class LibHttp(s_stormtypes.Lib):
196
196
  'default': None},
197
197
  {'name': 'timeout', 'type': 'int', 'desc': 'Total timeout for the request in seconds.',
198
198
  'default': 300},
199
- {'name': 'allow_redirects', 'type': 'bool', 'desc': 'If set to false, do not follow redirects.',
199
+ {'name': 'allow_redirects', 'type': 'boolean', 'desc': 'If set to false, do not follow redirects.',
200
200
  'default': True},
201
201
  {'name': 'fields', 'type': 'list',
202
202
  'desc': 'A list of info dictionaries containing the name, value or sha256, '
@@ -205,7 +205,7 @@ class LibHttp(s_stormtypes.Lib):
205
205
  'and the corresponding file will be uploaded as the value for '
206
206
  'the field.',
207
207
  'default': None},
208
- {'name': 'proxy', 'type': ['bool', 'str'],
208
+ {'name': 'proxy', 'type': ['boolean', 'str'],
209
209
  'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
210
210
  {'name': 'ssl_opts', 'type': 'dict',
211
211
  'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',
@@ -226,7 +226,7 @@ class LibHttp(s_stormtypes.Lib):
226
226
  'default': 300},
227
227
  {'name': 'params', 'type': 'dict', 'desc': 'Optional parameters which may be passed to the connection request.',
228
228
  'default': None},
229
- {'name': 'proxy', 'type': ['bool', 'str'],
229
+ {'name': 'proxy', 'type': ['boolean', 'str'],
230
230
  'desc': 'Configure proxy usage. See $lib.inet.http help for additional details.', 'default': True},
231
231
  {'name': 'ssl_opts', 'type': 'dict',
232
232
  'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.',