synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (89) hide show
  1. synapse/axon.py +3 -3
  2. synapse/cmds/cortex.py +1 -6
  3. synapse/common.py +7 -1
  4. synapse/cortex.py +145 -192
  5. synapse/datamodel.py +36 -1
  6. synapse/lib/agenda.py +87 -97
  7. synapse/lib/aha.py +51 -0
  8. synapse/lib/ast.py +22 -23
  9. synapse/lib/base.py +0 -6
  10. synapse/lib/boss.py +3 -0
  11. synapse/lib/cell.py +70 -39
  12. synapse/lib/certdir.py +9 -0
  13. synapse/lib/hiveauth.py +65 -12
  14. synapse/lib/httpapi.py +1 -0
  15. synapse/lib/modelrev.py +121 -33
  16. synapse/lib/modules.py +1 -0
  17. synapse/lib/nexus.py +64 -26
  18. synapse/lib/parser.py +2 -0
  19. synapse/lib/schemas.py +14 -0
  20. synapse/lib/snap.py +50 -4
  21. synapse/lib/storm.lark +4 -3
  22. synapse/lib/storm.py +96 -22
  23. synapse/lib/storm_format.py +1 -0
  24. synapse/lib/stormlib/aha.py +7 -1
  25. synapse/lib/stormlib/auth.py +13 -5
  26. synapse/lib/stormlib/cache.py +202 -0
  27. synapse/lib/stormlib/cortex.py +147 -8
  28. synapse/lib/stormlib/gen.py +53 -6
  29. synapse/lib/stormlib/math.py +1 -1
  30. synapse/lib/stormlib/model.py +11 -1
  31. synapse/lib/stormlib/spooled.py +109 -0
  32. synapse/lib/stormlib/vault.py +1 -1
  33. synapse/lib/stormtypes.py +113 -17
  34. synapse/lib/trigger.py +36 -47
  35. synapse/lib/types.py +29 -2
  36. synapse/lib/version.py +2 -2
  37. synapse/lib/view.py +80 -53
  38. synapse/models/economic.py +174 -5
  39. synapse/models/files.py +2 -0
  40. synapse/models/inet.py +77 -2
  41. synapse/models/infotech.py +12 -12
  42. synapse/models/orgs.py +72 -21
  43. synapse/models/person.py +40 -11
  44. synapse/models/risk.py +78 -24
  45. synapse/models/science.py +102 -0
  46. synapse/telepath.py +117 -35
  47. synapse/tests/test_cortex.py +84 -158
  48. synapse/tests/test_datamodel.py +22 -0
  49. synapse/tests/test_lib_agenda.py +52 -96
  50. synapse/tests/test_lib_aha.py +126 -4
  51. synapse/tests/test_lib_ast.py +412 -6
  52. synapse/tests/test_lib_cell.py +24 -8
  53. synapse/tests/test_lib_certdir.py +32 -0
  54. synapse/tests/test_lib_grammar.py +9 -1
  55. synapse/tests/test_lib_httpapi.py +0 -1
  56. synapse/tests/test_lib_jupyter.py +0 -1
  57. synapse/tests/test_lib_modelrev.py +41 -0
  58. synapse/tests/test_lib_nexus.py +38 -0
  59. synapse/tests/test_lib_storm.py +95 -5
  60. synapse/tests/test_lib_stormlib_cache.py +272 -0
  61. synapse/tests/test_lib_stormlib_cortex.py +71 -0
  62. synapse/tests/test_lib_stormlib_gen.py +37 -2
  63. synapse/tests/test_lib_stormlib_model.py +2 -0
  64. synapse/tests/test_lib_stormlib_spooled.py +190 -0
  65. synapse/tests/test_lib_stormlib_vault.py +12 -3
  66. synapse/tests/test_lib_stormsvc.py +0 -10
  67. synapse/tests/test_lib_stormtypes.py +60 -8
  68. synapse/tests/test_lib_trigger.py +20 -2
  69. synapse/tests/test_lib_types.py +17 -1
  70. synapse/tests/test_model_economic.py +114 -0
  71. synapse/tests/test_model_files.py +2 -0
  72. synapse/tests/test_model_inet.py +73 -1
  73. synapse/tests/test_model_infotech.py +2 -2
  74. synapse/tests/test_model_orgs.py +10 -1
  75. synapse/tests/test_model_risk.py +30 -2
  76. synapse/tests/test_model_science.py +59 -0
  77. synapse/tests/test_model_syn.py +0 -1
  78. synapse/tests/test_telepath.py +30 -7
  79. synapse/tests/test_tools_modrole.py +81 -0
  80. synapse/tests/test_tools_moduser.py +105 -0
  81. synapse/tools/modrole.py +59 -7
  82. synapse/tools/moduser.py +78 -10
  83. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/METADATA +2 -2
  84. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/RECORD +87 -83
  85. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/WHEEL +1 -1
  86. synapse/lib/provenance.py +0 -111
  87. synapse/tests/test_lib_provenance.py +0 -37
  88. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/LICENSE +0 -0
  89. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/top_level.txt +0 -0
synapse/lib/nexus.py CHANGED
@@ -12,9 +12,12 @@ import synapse.common as s_common
12
12
  import synapse.telepath as s_telepath
13
13
 
14
14
  import synapse.lib.base as s_base
15
+ import synapse.lib.version as s_version
15
16
 
16
17
  logger = logging.getLogger(__name__)
17
18
 
19
+ leaderversion = 'Leader is a higher version than we are.'
20
+
18
21
  # As a mirror follower, amount of time before giving up on a write request
19
22
  FOLLOWER_WRITE_WAIT_S = 30.0
20
23
 
@@ -86,7 +89,7 @@ class NexsRoot(s_base.Base):
86
89
  self.started = False
87
90
  self.celliden = self.cell.iden
88
91
  self.readonly = False
89
- self.readonlyreason = None
92
+ self.writeholds = set()
90
93
 
91
94
  self.applytask = None
92
95
  self.applylock = asyncio.Lock()
@@ -266,23 +269,47 @@ class NexsRoot(s_base.Base):
266
269
  except Exception:
267
270
  logger.exception('Exception while replaying log')
268
271
 
269
- def setReadOnly(self, readonly, reason=None):
270
- self.readonly = readonly
271
- self.readonlyreason = reason
272
+ async def addWriteHold(self, reason):
273
+
274
+ self.readonly = True
275
+
276
+ if reason not in self.writeholds:
277
+ logger.error(f'Entering Read-Only Mode: {reason}')
278
+ self.writeholds.add(reason)
279
+ return True
280
+
281
+ return False
282
+
283
+ async def delWriteHold(self, reason):
284
+
285
+ if reason in self.writeholds:
286
+
287
+ self.writeholds.remove(reason)
288
+
289
+ if not self.writeholds:
290
+ self.readonly = False
291
+
292
+ return True
293
+
294
+ return False
295
+
296
+ def reqNotReadOnly(self):
297
+
298
+ if not self.readonly:
299
+ return
300
+
301
+ # sets have stable order like dicts, so use the first message...
302
+ for reason in self.writeholds:
303
+ raise s_exc.IsReadOnly(mesg=reason)
304
+
305
+ mesg = 'Unable to issue Nexus events when readonly is set.'
306
+ raise s_exc.IsReadOnly(mesg=mesg)
272
307
 
273
308
  async def issue(self, nexsiden, event, args, kwargs, meta=None):
274
309
  '''
275
310
  If I'm not a follower, mutate, otherwise, ask the leader to make the change and wait for the follower loop
276
311
  to hand me the result through a future.
277
312
  '''
278
- assert self.started, 'Attempt to issue before nexsroot is started'
279
-
280
- if self.readonly:
281
- mesg = self.readonlyreason
282
- if mesg is None:
283
- mesg = 'Unable to issue Nexus events when readonly is set.'
284
-
285
- raise s_exc.IsReadOnly(mesg=mesg)
286
313
 
287
314
  # pick up a reference to avoid race when we eventually can promote
288
315
  client = self.client
@@ -290,6 +317,10 @@ class NexsRoot(s_base.Base):
290
317
  if client is None:
291
318
  return await self.eat(nexsiden, event, args, kwargs, meta)
292
319
 
320
+ # check here because we shouldn't be sending an edit upstream if we
321
+ # are in readonly mode because the mirror sync will never complete.
322
+ self.reqNotReadOnly()
323
+
293
324
  try:
294
325
  await client.waitready(timeout=FOLLOWER_WRITE_WAIT_S)
295
326
  except asyncio.TimeoutError:
@@ -315,6 +346,7 @@ class NexsRoot(s_base.Base):
315
346
  meta = {}
316
347
 
317
348
  async with self.applylock:
349
+ self.reqNotReadOnly()
318
350
  # Keep a reference to the shielded task to ensure it isn't GC'd
319
351
  self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta)))
320
352
  return await asyncio.shield(self.applytask)
@@ -470,7 +502,15 @@ class NexsRoot(s_base.Base):
470
502
  if features.get('dynmirror'):
471
503
  await proxy.readyToMirror()
472
504
 
473
- cellvers = cellinfo['synapse']['version']
505
+ cellvers = cellinfo['cell']['version']
506
+ if cellvers > self.cell.VERSION:
507
+ logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
508
+ await self.addWriteHold(leaderversion)
509
+ # this will fire again on reconnect...
510
+ return
511
+
512
+ # When we reconnect and the leader version has become ok...
513
+ await self.delWriteHold(leaderversion)
474
514
 
475
515
  if self.celliden is not None:
476
516
  if self.celliden != await proxy.getCellIden():
@@ -482,6 +522,11 @@ class NexsRoot(s_base.Base):
482
522
  while not proxy.isfini:
483
523
 
484
524
  try:
525
+
526
+ if self.readonly:
527
+ await self.waitfini(timeout=2)
528
+ continue
529
+
485
530
  offs = self.nexslog.index()
486
531
 
487
532
  opts = {}
@@ -491,14 +536,12 @@ class NexsRoot(s_base.Base):
491
536
  genr = proxy.getNexusChanges(offs, **opts)
492
537
  async for item in genr:
493
538
 
494
- if self.readonly:
495
- logger.error('Unable to consume Nexus events when readonly is set')
496
- await self.client.fini()
497
- return
498
-
499
539
  if proxy.isfini: # pragma: no cover
500
540
  break
501
541
 
542
+ if self.readonly:
543
+ break
544
+
502
545
  # with tellready we move to ready=true when we get a None
503
546
  if item is None:
504
547
  logger.info(f'Entering realtime change window for syncing data at offset={offs}')
@@ -519,9 +562,6 @@ class NexsRoot(s_base.Base):
519
562
  try:
520
563
  retn = await self.eat(*args)
521
564
 
522
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
523
- raise
524
-
525
565
  except Exception as e:
526
566
  if respfutu is not None:
527
567
  assert not respfutu.done()
@@ -533,9 +573,6 @@ class NexsRoot(s_base.Base):
533
573
  if respfutu is not None:
534
574
  respfutu.set_result(retn)
535
575
 
536
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
537
- raise
538
-
539
576
  except Exception: # pragma: no cover
540
577
  logger.exception('error in mirror loop')
541
578
 
@@ -552,10 +589,11 @@ class NexsRoot(s_base.Base):
552
589
 
553
590
  try:
554
591
  await self.cell.ahaclient.waitready(timeout=5)
555
- ahainfo = await self.cell.ahaclient.getCellInfo()
592
+ proxy = await self.cell.ahaclient.proxy(timeout=5)
593
+ ahainfo = await proxy.getCellInfo()
556
594
  ahavers = ahainfo['synapse']['version']
557
595
  if self.cell.ahasvcname is not None and ahavers >= (2, 95, 0):
558
- await self.cell.ahaclient.modAhaSvcInfo(self.cell.ahasvcname, {'ready': status})
596
+ await proxy.modAhaSvcInfo(self.cell.ahasvcname, {'ready': status})
559
597
 
560
598
  except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
561
599
  raise
synapse/lib/parser.py CHANGED
@@ -72,6 +72,7 @@ terminalEnglishMap = {
72
72
  'MODSET': '+= or -=',
73
73
  'NONQUOTEWORD': 'unquoted value',
74
74
  'NOT': 'not',
75
+ 'NULL': 'null',
75
76
  'NUMBER': 'number',
76
77
  'OCTNUMBER': 'number',
77
78
  'OR': 'or',
@@ -618,6 +619,7 @@ terminalClassMap = {
618
619
  'HEXNUMBER': lambda astinfo, x: s_ast.Const(astinfo, s_ast.parseNumber(x)),
619
620
  'OCTNUMBER': lambda astinfo, x: s_ast.Const(astinfo, s_ast.parseNumber(x)),
620
621
  'BOOL': lambda astinfo, x: s_ast.Bool(astinfo, x == 'true'),
622
+ 'NULL': lambda astinfo, x: s_ast.Const(astinfo, None),
621
623
  'SINGLEQUOTEDSTRING': lambda astinfo, x: s_ast.Const(astinfo, x[1:-1]), # drop quotes
622
624
  'NONQUOTEWORD': massage_vartokn,
623
625
  'VARTOKN': massage_vartokn,
synapse/lib/schemas.py CHANGED
@@ -40,6 +40,7 @@ _HttpExtAPIConfSchema = {
40
40
  'name': {'type': 'string', 'default': ''},
41
41
  'desc': {'type': 'string', 'default': ''},
42
42
  'path': {'type': 'string', 'minlen': 1},
43
+ 'pool': {'type': 'boolean', 'default': False},
43
44
  'view': {'type': 'string', 'pattern': s_config.re_iden},
44
45
  'runas': {'type': 'string', 'pattern': '^(owner|user)$'},
45
46
  'owner': {'type': 'string', 'pattern': s_config.re_iden},
@@ -93,6 +94,7 @@ _CronJobSchema = {
93
94
  'iden': {'type': 'string', 'pattern': s_config.re_iden},
94
95
  'view': {'type': 'string', 'pattern': s_config.re_iden},
95
96
  'name': {'type': 'string'},
97
+ 'pool': {'type': 'boolean'},
96
98
  'doc': {'type': 'string'},
97
99
  'incunit': {
98
100
  'oneOf': [
@@ -203,6 +205,7 @@ reqValidMerge = s_config.getJsValidator({
203
205
  'creator': {'type': 'string', 'pattern': s_config.re_iden},
204
206
  'created': {'type': 'number', 'minval': 0},
205
207
  'comment': {'type': 'string'},
208
+ 'updated': {'type': 'number', 'minval': 0},
206
209
  },
207
210
  'required': ['iden', 'creator', 'created'],
208
211
  'additionalProperties': False,
@@ -216,6 +219,7 @@ reqValidVote = s_config.getJsValidator({
216
219
  'created': {'type': 'number', 'minval': 0},
217
220
  'approved': {'type': 'boolean'},
218
221
  'comment': {'type': 'string'},
222
+ 'updated': {'type': 'number', 'minval': 0},
219
223
  },
220
224
  'required': ['user', 'offset', 'created', 'approved'],
221
225
  'additionalProperties': False,
@@ -273,3 +277,13 @@ reqValidSslCtxOpts = s_config.getJsValidator({
273
277
  },
274
278
  'additionalProperties': False,
275
279
  })
280
+
281
+ _stormPoolOptsSchema = {
282
+ 'type': 'object',
283
+ 'properties': {
284
+ 'timeout:sync': {'type': 'integer', 'minimum': 1},
285
+ 'timeout:connection': {'type': 'integer', 'minimum': 1},
286
+ },
287
+ 'additionalProperties': False,
288
+ }
289
+ reqValidStormPoolOpts = s_config.getJsValidator(_stormPoolOptsSchema)
synapse/lib/snap.py CHANGED
@@ -209,6 +209,27 @@ class ProtoNode:
209
209
  if tagnode is not s_common.novalu:
210
210
  return self.ctx.loadNode(tagnode)
211
211
 
212
+ # check for an :isnow tag redirection in our hierarchy...
213
+ toks = info.get('toks')
214
+ for i in range(len(toks)):
215
+
216
+ toktag = '.'.join(toks[:i + 1])
217
+ toknode = await self.ctx.snap.getTagNode(toktag)
218
+ if toknode is s_common.novalu:
219
+ continue
220
+
221
+ tokvalu = toknode.ndef[1]
222
+ if tokvalu == toktag:
223
+ continue
224
+
225
+ realnow = tokvalu + norm[len(toktag):]
226
+ tagnode = await self.ctx.snap.getTagNode(realnow)
227
+ if tagnode is not s_common.novalu:
228
+ return self.ctx.loadNode(tagnode)
229
+
230
+ norm, info = await self.ctx.snap.getTagNorm(realnow)
231
+ break
232
+
212
233
  return await self.ctx.addNode('syn:tag', norm, norminfo=info)
213
234
 
214
235
  def getTag(self, tag):
@@ -619,10 +640,6 @@ class Snap(s_base.Base):
619
640
 
620
641
  self.changelog = []
621
642
  self.tagtype = self.core.model.type('ival')
622
- self.trigson = self.core.trigson
623
-
624
- def disableTriggers(self):
625
- self.trigson = False
626
643
 
627
644
  async def getSnapMeta(self):
628
645
  '''
@@ -661,6 +678,33 @@ class Snap(s_base.Base):
661
678
  self.onfini(runt)
662
679
  return runt
663
680
 
681
+ async def _joinEmbedStor(self, storage, embeds):
682
+ for nodePath, relProps in embeds.items():
683
+ await asyncio.sleep(0)
684
+ iden = relProps.get('*')
685
+ if not iden:
686
+ continue
687
+
688
+ stor = await self.view.getStorNodes(s_common.uhex(iden))
689
+ for relProp in relProps.keys():
690
+ await asyncio.sleep(0)
691
+ if relProp == '*':
692
+ continue
693
+
694
+ for idx, layrstor in enumerate(stor):
695
+ await asyncio.sleep(0)
696
+ props = layrstor.get('props')
697
+ if not props:
698
+ continue
699
+
700
+ if relProp not in props:
701
+ continue
702
+
703
+ if 'embeds' not in storage[idx]:
704
+ storage[idx]['embeds'] = {}
705
+
706
+ storage[idx]['embeds'][f'{nodePath}::{relProp}'] = props[relProp]
707
+
664
708
  async def iterStormPodes(self, text, opts, user=None):
665
709
  '''
666
710
  Yield packed node tuples for the given storm query text.
@@ -703,6 +747,8 @@ class Snap(s_base.Base):
703
747
  embdef = embeds.get(node.form.name)
704
748
  if embdef is not None:
705
749
  pode[1]['embeds'] = await node.getEmbeds(embdef)
750
+ if show_storage:
751
+ await self._joinEmbedStor(pode[1]['storage'], pode[1]['embeds'])
706
752
 
707
753
  yield pode
708
754
 
synapse/lib/storm.lark CHANGED
@@ -56,8 +56,8 @@ edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS
56
56
  edittagpropdel: EXPRMINUS tagprop
57
57
  EQSPACE: /((?<=\s)=|=(?=\s))/
58
58
  MODSET.4: "+=" | "-="
59
- TRYSETPLUS: "?+="
60
- TRYSETMINUS: "?-="
59
+ TRYSETPLUS.1: "?+="
60
+ TRYSETMINUS.1: "?-="
61
61
  TRYSET.1: "?="
62
62
  SETTAGOPER: "?"
63
63
 
@@ -308,7 +308,7 @@ evalvalu: _valu
308
308
  exprdict: "{" ((_exprvalu | VARTOKN) (":" | _EXPRCOLONNOSPACE) (_exprvalu | VARTOKN) ("," (_exprvalu | VARTOKN) (":" | _EXPRCOLONNOSPACE) (_exprvalu | VARTOKN))* ","? )? "}"
309
309
  exprlist: "[" ((_exprvalu | VARTOKN) ("," (_exprvalu | VARTOKN))* ","? )? "]"
310
310
  // Just like _valu, but doesn't allow valu lists or unquoted strings or queries
311
- _exprvalu: NUMBER | HEXNUMBER | OCTNUMBER | BOOL | exprlist | exprdict | _exprvarvalu | exprrelpropvalu
311
+ _exprvalu: NUMBER | HEXNUMBER | OCTNUMBER | BOOL | NULL | exprlist | exprdict | _exprvarvalu | exprrelpropvalu
312
312
  | exprunivpropvalu | exprtagvalu | exprtagpropvalu | TRIPLEQUOTEDSTRING | DOUBLEQUOTEDSTRING
313
313
  | SINGLEQUOTEDSTRING | formatstring | _innerdollarexprs
314
314
 
@@ -590,6 +590,7 @@ OCTNUMBER.1: /
590
590
  /x
591
591
 
592
592
  BOOL.2: /(true|false)(?=$|[\s\),\]}\|\=])/
593
+ NULL.2: "null"
593
594
  NOT.2: "not"
594
595
  OR.2: "or"
595
596
  AND.2: "and"
synapse/lib/storm.py CHANGED
@@ -27,8 +27,8 @@ import synapse.lib.msgpack as s_msgpack
27
27
  import synapse.lib.spooled as s_spooled
28
28
  import synapse.lib.version as s_version
29
29
  import synapse.lib.hashitem as s_hashitem
30
+ import synapse.lib.hiveauth as s_hiveauth
30
31
  import synapse.lib.stormctrl as s_stormctrl
31
- import synapse.lib.provenance as s_provenance
32
32
  import synapse.lib.stormtypes as s_stormtypes
33
33
 
34
34
  import synapse.lib.stormlib.graph as s_stormlib_graph
@@ -225,6 +225,7 @@ reqValidPkgdef = s_config.getJsValidator({
225
225
  },
226
226
  'required': ['cert', 'sign'],
227
227
  },
228
+ # TODO: Remove me after Synapse 3.0.0.
228
229
  'synapse_minversion': {
229
230
  'type': ['array', 'null'],
230
231
  'items': {'type': 'number'}
@@ -1206,6 +1207,8 @@ stormcmds = (
1206
1207
  'descr': addcrondescr,
1207
1208
  'cmdargs': (
1208
1209
  ('query', {'help': 'Query for the cron job to execute.'}),
1210
+ ('--pool', {'action': 'store_true', 'default': False,
1211
+ 'help': 'Allow the cron job to be run by a mirror from the query pool.'}),
1209
1212
  ('--minute', {'help': 'Minute value for job or recurrence period.'}),
1210
1213
  ('--name', {'help': 'An optional name for the cron job.'}),
1211
1214
  ('--doc', {'help': 'An optional doc string for the cron job.'}),
@@ -1225,6 +1228,7 @@ stormcmds = (
1225
1228
  minute=$cmdopts.minute,
1226
1229
  hour=$cmdopts.hour,
1227
1230
  day=$cmdopts.day,
1231
+ pool=$cmdopts.pool,
1228
1232
  month=$cmdopts.month,
1229
1233
  year=$cmdopts.year,
1230
1234
  hourly=$cmdopts.hourly,
@@ -1372,6 +1376,7 @@ stormcmds = (
1372
1376
  $lib.print('iden: {iden}', iden=$job.iden)
1373
1377
  $lib.print('user: {user}', user=$job.user)
1374
1378
  $lib.print('enabled: {enabled}', enabled=$job.enabled)
1379
+ $lib.print(`pool: {$job.pool}`)
1375
1380
  $lib.print('recurring: {isrecur}', isrecur=$job.isrecur)
1376
1381
  $lib.print('# starts: {startcount}', startcount=$job.startcount)
1377
1382
  $lib.print('# errors: {errcount}', errcount=$job.errcount)
@@ -1824,6 +1829,8 @@ class Runtime(s_base.Base):
1824
1829
  opaque object which is called through, but not dereferenced.
1825
1830
 
1826
1831
  '''
1832
+
1833
+ _admin_reason = s_hiveauth._allowedReason(True, isadmin=True)
1827
1834
  async def __anit__(self, query, snap, opts=None, user=None, root=None):
1828
1835
 
1829
1836
  await s_base.Base.__anit__(self)
@@ -2187,25 +2194,93 @@ class Runtime(s_base.Base):
2187
2194
 
2188
2195
  return self.user.allowed(perms, gateiden=gateiden, default=default)
2189
2196
 
2197
+ def allowedReason(self, perms, gateiden=None, default=None):
2198
+ '''
2199
+ Similar to allowed, but always prefer the default value specified by the caller.
2200
+ Default values are still pulled from permdefs if there is a match there; but still prefer caller default.
2201
+ This results in a ternary response that can be used to know if a rule had a positive/negative or no match.
2202
+ The matching reason metadata is also returned.
2203
+ '''
2204
+ if self.asroot:
2205
+ return self._admin_reason
2206
+
2207
+ if default is None:
2208
+ permdef = self.snap.core.getPermDef(perms)
2209
+ if permdef:
2210
+ default = permdef.get('default', default)
2211
+
2212
+ return self.user.getAllowedReason(perms, gateiden=gateiden, default=default)
2213
+
2190
2214
  def confirmPropSet(self, prop, layriden=None):
2191
2215
 
2192
2216
  if layriden is None:
2193
2217
  layriden = self.snap.wlyr.iden
2194
2218
 
2195
- if any(self.allowed(perm, gateiden=self.snap.wlyr.iden) for perm in prop.setperms):
2219
+ meta0 = self.allowedReason(prop.setperms[0], gateiden=layriden)
2220
+
2221
+ if meta0.isadmin:
2196
2222
  return
2197
2223
 
2198
- self.user.raisePermDeny(prop.setperms[-1], gateiden=layriden)
2224
+ allowed0 = meta0.value
2225
+
2226
+ meta1 = self.allowedReason(prop.setperms[1], gateiden=layriden)
2227
+ allowed1 = meta1.value
2228
+
2229
+ if allowed0:
2230
+ if allowed1:
2231
+ return
2232
+ elif allowed1 is False:
2233
+ # This is a allow-with-precedence case.
2234
+ # Inspect meta to determine if the rule a0 is more specific than rule a1
2235
+ if len(meta0.rule) >= len(meta1.rule):
2236
+ return
2237
+ self.user.raisePermDeny(prop.setperms[0], gateiden=layriden)
2238
+ return
2239
+
2240
+ if allowed1:
2241
+ if allowed0 is None:
2242
+ return
2243
+ # allowed0 here is False. This is a deny-with-precedence case.
2244
+ # Inspect meta to determine if the rule a1 is more specific than rule a0
2245
+ if len(meta1.rule) > len(meta0.rule):
2246
+ return
2247
+
2248
+ self.user.raisePermDeny(prop.setperms[0], gateiden=layriden)
2199
2249
 
2200
2250
  def confirmPropDel(self, prop, layriden=None):
2201
2251
 
2202
2252
  if layriden is None:
2203
2253
  layriden = self.snap.wlyr.iden
2204
2254
 
2205
- if any(self.allowed(perm, gateiden=self.snap.wlyr.iden) for perm in prop.delperms):
2255
+ meta0 = self.allowedReason(prop.delperms[0], gateiden=layriden)
2256
+
2257
+ if meta0.isadmin:
2258
+ return
2259
+
2260
+ allowed0 = meta0.value
2261
+ meta1 = self.allowedReason(prop.delperms[1], gateiden=layriden)
2262
+ allowed1 = meta1.value
2263
+
2264
+ if allowed0:
2265
+ if allowed1:
2266
+ return
2267
+ elif allowed1 is False:
2268
+ # This is a allow-with-precedence case.
2269
+ # Inspect meta to determine if the rule a0 is more specific than rule a1
2270
+ if len(meta0.rule) >= len(meta1.rule):
2271
+ return
2272
+ self.user.raisePermDeny(prop.delperms[0], gateiden=layriden)
2206
2273
  return
2207
2274
 
2208
- self.user.raisePermDeny(prop.delperms[-1], gateiden=layriden)
2275
+ if allowed1:
2276
+ if allowed0 is None:
2277
+ return
2278
+ # allowed0 here is False. This is a deny-with-precedence case.
2279
+ # Inspect meta to determine if the rule a1 is more specific than rule a0
2280
+ if len(meta1.rule) > len(meta0.rule):
2281
+ return
2282
+
2283
+ self.user.raisePermDeny(prop.delperms[0], gateiden=layriden)
2209
2284
 
2210
2285
  def confirmEasyPerm(self, item, perm, mesg=None):
2211
2286
  if not self.asroot:
@@ -2238,27 +2313,26 @@ class Runtime(s_base.Base):
2238
2313
 
2239
2314
  async def execute(self, genr=None):
2240
2315
  try:
2241
- with s_provenance.claim('storm', q=self.query.text, user=self.user.iden):
2242
2316
 
2243
- async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
2244
- nodegenr, empty = await s_ast.pullone(nodegenr)
2317
+ async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
2318
+ nodegenr, empty = await s_ast.pullone(nodegenr)
2245
2319
 
2246
- if empty:
2247
- return
2320
+ if empty:
2321
+ return
2248
2322
 
2249
- rules = self.opts.get('graph')
2250
- if rules not in (False, None):
2251
- if rules is True:
2252
- rules = {'degrees': None, 'refs': True}
2253
- elif isinstance(rules, str):
2254
- rules = await self.snap.core.getStormGraph(rules)
2323
+ rules = self.opts.get('graph')
2324
+ if rules not in (False, None):
2325
+ if rules is True:
2326
+ rules = {'degrees': None, 'refs': True}
2327
+ elif isinstance(rules, str):
2328
+ rules = await self.snap.core.getStormGraph(rules)
2255
2329
 
2256
- subgraph = s_ast.SubGraph(rules)
2257
- nodegenr = subgraph.run(self, nodegenr)
2330
+ subgraph = s_ast.SubGraph(rules)
2331
+ nodegenr = subgraph.run(self, nodegenr)
2258
2332
 
2259
- async for item in nodegenr:
2260
- self.tick()
2261
- yield item
2333
+ async for item in nodegenr:
2334
+ self.tick()
2335
+ yield item
2262
2336
 
2263
2337
  except RecursionError:
2264
2338
  mesg = 'Maximum Storm pipeline depth exceeded.'
@@ -3589,7 +3663,7 @@ class CopyToCmd(Cmd):
3589
3663
 
3590
3664
  runt.confirm(node.form.addperm, gateiden=layriden)
3591
3665
  for name in node.props.keys():
3592
- runt.confirmPropSet(node.form.props[name])
3666
+ runt.confirmPropSet(node.form.props[name], layriden=layriden)
3593
3667
 
3594
3668
  for tag in node.tags.keys():
3595
3669
  runt.confirm(('node', 'tag', 'add', *tag.split('.')), gateiden=layriden)
@@ -55,6 +55,7 @@ TerminalPygMap = {
55
55
  'MODSET': p_t.Operator,
56
56
  'NONQUOTEWORD': p_t.Literal,
57
57
  'NOT': p_t.Keyword,
58
+ 'NULL': p_t.Keyword,
58
59
  'NUMBER': p_t.Literal.Number,
59
60
  'OCTNUMBER': p_t.Literal.Number,
60
61
  'OR': p_t.Keyword,
@@ -35,6 +35,7 @@ class AhaPoolLib(s_stormtypes.Lib):
35
35
  if poolinfo is not None:
36
36
  return AhaPool(self.runt, poolinfo)
37
37
 
38
+ @s_stormtypes.stormfunc(readonly=True)
38
39
  async def list(self):
39
40
 
40
41
  self.runt.reqAdmin()
@@ -45,6 +46,9 @@ class AhaPoolLib(s_stormtypes.Lib):
45
46
 
46
47
  @s_stormtypes.registry.registerType
47
48
  class AhaPool(s_stormtypes.StormType):
49
+ '''
50
+ Implements the Storm API for an AHA pool.
51
+ '''
48
52
  _storm_typename = 'aha:pool'
49
53
 
50
54
  def __init__(self, runt, poolinfo):
@@ -57,6 +61,9 @@ class AhaPool(s_stormtypes.StormType):
57
61
  'del': self._del,
58
62
  })
59
63
 
64
+ async def stormrepr(self):
65
+ return f'{self._storm_typename}: {self.poolinfo.get("name")}'
66
+
60
67
  async def _derefGet(self, name):
61
68
  return self.poolinfo.get(name)
62
69
 
@@ -94,7 +101,6 @@ stormcmds = (
94
101
  for $pool in $lib.aha.pool.list() {
95
102
  $count = ($count + 1)
96
103
  $lib.print(`Pool: {$pool.name}`)
97
- $lib.print($pool)
98
104
  for ($svcname, $svcinfo) in $pool.services {
99
105
  $lib.print(` {$svcname}`)
100
106
  }
@@ -1192,7 +1192,8 @@ class User(s_stormtypes.Prim):
1192
1192
 
1193
1193
  perm = tuple(permname.split('.'))
1194
1194
  user = await self.runt.snap.core.auth.reqUser(self.valu)
1195
- return user.getAllowedReason(perm, gateiden=gateiden, default=default)
1195
+ reason = user.getAllowedReason(perm, gateiden=gateiden, default=default)
1196
+ return reason.value, reason.mesg
1196
1197
 
1197
1198
  async def _methUserGrant(self, iden, indx=None):
1198
1199
  self.runt.confirm(('auth', 'user', 'grant'))
@@ -1226,6 +1227,11 @@ class User(s_stormtypes.Prim):
1226
1227
  indx = await s_stormtypes.toint(indx, noneok=True)
1227
1228
  gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
1228
1229
  self.runt.confirm(('auth', 'user', 'set', 'rules'), gateiden=gateiden)
1230
+ # TODO: Remove me in 3.0.0
1231
+ if gateiden == 'cortex':
1232
+ mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
1233
+ f'for permission checks and will be removed in Synapse v3.0.0.'
1234
+ await self.runt.snap.warn(mesg, log=False)
1229
1235
  await self.runt.snap.core.addUserRule(self.valu, rule, indx=indx, gateiden=gateiden)
1230
1236
 
1231
1237
  async def _methUserDelRule(self, rule, gateiden=None):
@@ -1479,6 +1485,11 @@ class Role(s_stormtypes.Prim):
1479
1485
  indx = await s_stormtypes.toint(indx, noneok=True)
1480
1486
  gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
1481
1487
  self.runt.confirm(('auth', 'role', 'set', 'rules'), gateiden=gateiden)
1488
+ # TODO: Remove me in 3.0.0
1489
+ if gateiden == 'cortex':
1490
+ mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
1491
+ f'for permission checks and will be removed in Synapse v3.0.0.'
1492
+ await self.runt.snap.warn(mesg, log=False)
1482
1493
  await self.runt.snap.core.addRoleRule(self.valu, rule, indx=indx, gateiden=gateiden)
1483
1494
 
1484
1495
  async def _methRoleDelRule(self, rule, gateiden=None):
@@ -1557,10 +1568,7 @@ class LibAuth(s_stormtypes.Lib):
1557
1568
  @s_stormtypes.stormfunc(readonly=True)
1558
1569
  async def textFromRule(self, rule):
1559
1570
  rule = await s_stormtypes.toprim(rule)
1560
- text = '.'.join(rule[1])
1561
- if not rule[0]:
1562
- text = '!' + text
1563
- return text
1571
+ return s_common.reprauthrule(rule)
1564
1572
 
1565
1573
  @s_stormtypes.stormfunc(readonly=True)
1566
1574
  async def getPermDefs(self):