synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +3 -3
- synapse/cmds/cortex.py +1 -6
- synapse/common.py +7 -1
- synapse/cortex.py +145 -192
- synapse/datamodel.py +36 -1
- synapse/lib/agenda.py +87 -97
- synapse/lib/aha.py +51 -0
- synapse/lib/ast.py +22 -23
- synapse/lib/base.py +0 -6
- synapse/lib/boss.py +3 -0
- synapse/lib/cell.py +70 -39
- synapse/lib/certdir.py +9 -0
- synapse/lib/hiveauth.py +65 -12
- synapse/lib/httpapi.py +1 -0
- synapse/lib/modelrev.py +121 -33
- synapse/lib/modules.py +1 -0
- synapse/lib/nexus.py +64 -26
- synapse/lib/parser.py +2 -0
- synapse/lib/schemas.py +14 -0
- synapse/lib/snap.py +50 -4
- synapse/lib/storm.lark +4 -3
- synapse/lib/storm.py +96 -22
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormlib/aha.py +7 -1
- synapse/lib/stormlib/auth.py +13 -5
- synapse/lib/stormlib/cache.py +202 -0
- synapse/lib/stormlib/cortex.py +147 -8
- synapse/lib/stormlib/gen.py +53 -6
- synapse/lib/stormlib/math.py +1 -1
- synapse/lib/stormlib/model.py +11 -1
- synapse/lib/stormlib/spooled.py +109 -0
- synapse/lib/stormlib/vault.py +1 -1
- synapse/lib/stormtypes.py +113 -17
- synapse/lib/trigger.py +36 -47
- synapse/lib/types.py +29 -2
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +80 -53
- synapse/models/economic.py +174 -5
- synapse/models/files.py +2 -0
- synapse/models/inet.py +77 -2
- synapse/models/infotech.py +12 -12
- synapse/models/orgs.py +72 -21
- synapse/models/person.py +40 -11
- synapse/models/risk.py +78 -24
- synapse/models/science.py +102 -0
- synapse/telepath.py +117 -35
- synapse/tests/test_cortex.py +84 -158
- synapse/tests/test_datamodel.py +22 -0
- synapse/tests/test_lib_agenda.py +52 -96
- synapse/tests/test_lib_aha.py +126 -4
- synapse/tests/test_lib_ast.py +412 -6
- synapse/tests/test_lib_cell.py +24 -8
- synapse/tests/test_lib_certdir.py +32 -0
- synapse/tests/test_lib_grammar.py +9 -1
- synapse/tests/test_lib_httpapi.py +0 -1
- synapse/tests/test_lib_jupyter.py +0 -1
- synapse/tests/test_lib_modelrev.py +41 -0
- synapse/tests/test_lib_nexus.py +38 -0
- synapse/tests/test_lib_storm.py +95 -5
- synapse/tests/test_lib_stormlib_cache.py +272 -0
- synapse/tests/test_lib_stormlib_cortex.py +71 -0
- synapse/tests/test_lib_stormlib_gen.py +37 -2
- synapse/tests/test_lib_stormlib_model.py +2 -0
- synapse/tests/test_lib_stormlib_spooled.py +190 -0
- synapse/tests/test_lib_stormlib_vault.py +12 -3
- synapse/tests/test_lib_stormsvc.py +0 -10
- synapse/tests/test_lib_stormtypes.py +60 -8
- synapse/tests/test_lib_trigger.py +20 -2
- synapse/tests/test_lib_types.py +17 -1
- synapse/tests/test_model_economic.py +114 -0
- synapse/tests/test_model_files.py +2 -0
- synapse/tests/test_model_inet.py +73 -1
- synapse/tests/test_model_infotech.py +2 -2
- synapse/tests/test_model_orgs.py +10 -1
- synapse/tests/test_model_risk.py +30 -2
- synapse/tests/test_model_science.py +59 -0
- synapse/tests/test_model_syn.py +0 -1
- synapse/tests/test_telepath.py +30 -7
- synapse/tests/test_tools_modrole.py +81 -0
- synapse/tests/test_tools_moduser.py +105 -0
- synapse/tools/modrole.py +59 -7
- synapse/tools/moduser.py +78 -10
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/METADATA +2 -2
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/RECORD +87 -83
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/WHEEL +1 -1
- synapse/lib/provenance.py +0 -111
- synapse/tests/test_lib_provenance.py +0 -37
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/LICENSE +0 -0
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/top_level.txt +0 -0
synapse/lib/nexus.py
CHANGED
|
@@ -12,9 +12,12 @@ import synapse.common as s_common
|
|
|
12
12
|
import synapse.telepath as s_telepath
|
|
13
13
|
|
|
14
14
|
import synapse.lib.base as s_base
|
|
15
|
+
import synapse.lib.version as s_version
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
17
18
|
|
|
19
|
+
leaderversion = 'Leader is a higher version than we are.'
|
|
20
|
+
|
|
18
21
|
# As a mirror follower, amount of time before giving up on a write request
|
|
19
22
|
FOLLOWER_WRITE_WAIT_S = 30.0
|
|
20
23
|
|
|
@@ -86,7 +89,7 @@ class NexsRoot(s_base.Base):
|
|
|
86
89
|
self.started = False
|
|
87
90
|
self.celliden = self.cell.iden
|
|
88
91
|
self.readonly = False
|
|
89
|
-
self.
|
|
92
|
+
self.writeholds = set()
|
|
90
93
|
|
|
91
94
|
self.applytask = None
|
|
92
95
|
self.applylock = asyncio.Lock()
|
|
@@ -266,23 +269,47 @@ class NexsRoot(s_base.Base):
|
|
|
266
269
|
except Exception:
|
|
267
270
|
logger.exception('Exception while replaying log')
|
|
268
271
|
|
|
269
|
-
def
|
|
270
|
-
|
|
271
|
-
self.
|
|
272
|
+
async def addWriteHold(self, reason):
|
|
273
|
+
|
|
274
|
+
self.readonly = True
|
|
275
|
+
|
|
276
|
+
if reason not in self.writeholds:
|
|
277
|
+
logger.error(f'Entering Read-Only Mode: {reason}')
|
|
278
|
+
self.writeholds.add(reason)
|
|
279
|
+
return True
|
|
280
|
+
|
|
281
|
+
return False
|
|
282
|
+
|
|
283
|
+
async def delWriteHold(self, reason):
|
|
284
|
+
|
|
285
|
+
if reason in self.writeholds:
|
|
286
|
+
|
|
287
|
+
self.writeholds.remove(reason)
|
|
288
|
+
|
|
289
|
+
if not self.writeholds:
|
|
290
|
+
self.readonly = False
|
|
291
|
+
|
|
292
|
+
return True
|
|
293
|
+
|
|
294
|
+
return False
|
|
295
|
+
|
|
296
|
+
def reqNotReadOnly(self):
|
|
297
|
+
|
|
298
|
+
if not self.readonly:
|
|
299
|
+
return
|
|
300
|
+
|
|
301
|
+
# sets have stable order like dicts, so use the first message...
|
|
302
|
+
for reason in self.writeholds:
|
|
303
|
+
raise s_exc.IsReadOnly(mesg=reason)
|
|
304
|
+
|
|
305
|
+
mesg = 'Unable to issue Nexus events when readonly is set.'
|
|
306
|
+
raise s_exc.IsReadOnly(mesg=mesg)
|
|
272
307
|
|
|
273
308
|
async def issue(self, nexsiden, event, args, kwargs, meta=None):
|
|
274
309
|
'''
|
|
275
310
|
If I'm not a follower, mutate, otherwise, ask the leader to make the change and wait for the follower loop
|
|
276
311
|
to hand me the result through a future.
|
|
277
312
|
'''
|
|
278
|
-
assert self.started, 'Attempt to issue before nexsroot is started'
|
|
279
|
-
|
|
280
|
-
if self.readonly:
|
|
281
|
-
mesg = self.readonlyreason
|
|
282
|
-
if mesg is None:
|
|
283
|
-
mesg = 'Unable to issue Nexus events when readonly is set.'
|
|
284
|
-
|
|
285
|
-
raise s_exc.IsReadOnly(mesg=mesg)
|
|
286
313
|
|
|
287
314
|
# pick up a reference to avoid race when we eventually can promote
|
|
288
315
|
client = self.client
|
|
@@ -290,6 +317,10 @@ class NexsRoot(s_base.Base):
|
|
|
290
317
|
if client is None:
|
|
291
318
|
return await self.eat(nexsiden, event, args, kwargs, meta)
|
|
292
319
|
|
|
320
|
+
# check here because we shouldn't be sending an edit upstream if we
|
|
321
|
+
# are in readonly mode because the mirror sync will never complete.
|
|
322
|
+
self.reqNotReadOnly()
|
|
323
|
+
|
|
293
324
|
try:
|
|
294
325
|
await client.waitready(timeout=FOLLOWER_WRITE_WAIT_S)
|
|
295
326
|
except asyncio.TimeoutError:
|
|
@@ -315,6 +346,7 @@ class NexsRoot(s_base.Base):
|
|
|
315
346
|
meta = {}
|
|
316
347
|
|
|
317
348
|
async with self.applylock:
|
|
349
|
+
self.reqNotReadOnly()
|
|
318
350
|
# Keep a reference to the shielded task to ensure it isn't GC'd
|
|
319
351
|
self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta)))
|
|
320
352
|
return await asyncio.shield(self.applytask)
|
|
@@ -470,7 +502,15 @@ class NexsRoot(s_base.Base):
|
|
|
470
502
|
if features.get('dynmirror'):
|
|
471
503
|
await proxy.readyToMirror()
|
|
472
504
|
|
|
473
|
-
cellvers = cellinfo['
|
|
505
|
+
cellvers = cellinfo['cell']['version']
|
|
506
|
+
if cellvers > self.cell.VERSION:
|
|
507
|
+
logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
|
|
508
|
+
await self.addWriteHold(leaderversion)
|
|
509
|
+
# this will fire again on reconnect...
|
|
510
|
+
return
|
|
511
|
+
|
|
512
|
+
# When we reconnect and the leader version has become ok...
|
|
513
|
+
await self.delWriteHold(leaderversion)
|
|
474
514
|
|
|
475
515
|
if self.celliden is not None:
|
|
476
516
|
if self.celliden != await proxy.getCellIden():
|
|
@@ -482,6 +522,11 @@ class NexsRoot(s_base.Base):
|
|
|
482
522
|
while not proxy.isfini:
|
|
483
523
|
|
|
484
524
|
try:
|
|
525
|
+
|
|
526
|
+
if self.readonly:
|
|
527
|
+
await self.waitfini(timeout=2)
|
|
528
|
+
continue
|
|
529
|
+
|
|
485
530
|
offs = self.nexslog.index()
|
|
486
531
|
|
|
487
532
|
opts = {}
|
|
@@ -491,14 +536,12 @@ class NexsRoot(s_base.Base):
|
|
|
491
536
|
genr = proxy.getNexusChanges(offs, **opts)
|
|
492
537
|
async for item in genr:
|
|
493
538
|
|
|
494
|
-
if self.readonly:
|
|
495
|
-
logger.error('Unable to consume Nexus events when readonly is set')
|
|
496
|
-
await self.client.fini()
|
|
497
|
-
return
|
|
498
|
-
|
|
499
539
|
if proxy.isfini: # pragma: no cover
|
|
500
540
|
break
|
|
501
541
|
|
|
542
|
+
if self.readonly:
|
|
543
|
+
break
|
|
544
|
+
|
|
502
545
|
# with tellready we move to ready=true when we get a None
|
|
503
546
|
if item is None:
|
|
504
547
|
logger.info(f'Entering realtime change window for syncing data at offset={offs}')
|
|
@@ -519,9 +562,6 @@ class NexsRoot(s_base.Base):
|
|
|
519
562
|
try:
|
|
520
563
|
retn = await self.eat(*args)
|
|
521
564
|
|
|
522
|
-
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
|
|
523
|
-
raise
|
|
524
|
-
|
|
525
565
|
except Exception as e:
|
|
526
566
|
if respfutu is not None:
|
|
527
567
|
assert not respfutu.done()
|
|
@@ -533,9 +573,6 @@ class NexsRoot(s_base.Base):
|
|
|
533
573
|
if respfutu is not None:
|
|
534
574
|
respfutu.set_result(retn)
|
|
535
575
|
|
|
536
|
-
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
|
|
537
|
-
raise
|
|
538
|
-
|
|
539
576
|
except Exception: # pragma: no cover
|
|
540
577
|
logger.exception('error in mirror loop')
|
|
541
578
|
|
|
@@ -552,10 +589,11 @@ class NexsRoot(s_base.Base):
|
|
|
552
589
|
|
|
553
590
|
try:
|
|
554
591
|
await self.cell.ahaclient.waitready(timeout=5)
|
|
555
|
-
|
|
592
|
+
proxy = await self.cell.ahaclient.proxy(timeout=5)
|
|
593
|
+
ahainfo = await proxy.getCellInfo()
|
|
556
594
|
ahavers = ahainfo['synapse']['version']
|
|
557
595
|
if self.cell.ahasvcname is not None and ahavers >= (2, 95, 0):
|
|
558
|
-
await
|
|
596
|
+
await proxy.modAhaSvcInfo(self.cell.ahasvcname, {'ready': status})
|
|
559
597
|
|
|
560
598
|
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
|
|
561
599
|
raise
|
synapse/lib/parser.py
CHANGED
|
@@ -72,6 +72,7 @@ terminalEnglishMap = {
|
|
|
72
72
|
'MODSET': '+= or -=',
|
|
73
73
|
'NONQUOTEWORD': 'unquoted value',
|
|
74
74
|
'NOT': 'not',
|
|
75
|
+
'NULL': 'null',
|
|
75
76
|
'NUMBER': 'number',
|
|
76
77
|
'OCTNUMBER': 'number',
|
|
77
78
|
'OR': 'or',
|
|
@@ -618,6 +619,7 @@ terminalClassMap = {
|
|
|
618
619
|
'HEXNUMBER': lambda astinfo, x: s_ast.Const(astinfo, s_ast.parseNumber(x)),
|
|
619
620
|
'OCTNUMBER': lambda astinfo, x: s_ast.Const(astinfo, s_ast.parseNumber(x)),
|
|
620
621
|
'BOOL': lambda astinfo, x: s_ast.Bool(astinfo, x == 'true'),
|
|
622
|
+
'NULL': lambda astinfo, x: s_ast.Const(astinfo, None),
|
|
621
623
|
'SINGLEQUOTEDSTRING': lambda astinfo, x: s_ast.Const(astinfo, x[1:-1]), # drop quotes
|
|
622
624
|
'NONQUOTEWORD': massage_vartokn,
|
|
623
625
|
'VARTOKN': massage_vartokn,
|
synapse/lib/schemas.py
CHANGED
|
@@ -40,6 +40,7 @@ _HttpExtAPIConfSchema = {
|
|
|
40
40
|
'name': {'type': 'string', 'default': ''},
|
|
41
41
|
'desc': {'type': 'string', 'default': ''},
|
|
42
42
|
'path': {'type': 'string', 'minlen': 1},
|
|
43
|
+
'pool': {'type': 'boolean', 'default': False},
|
|
43
44
|
'view': {'type': 'string', 'pattern': s_config.re_iden},
|
|
44
45
|
'runas': {'type': 'string', 'pattern': '^(owner|user)$'},
|
|
45
46
|
'owner': {'type': 'string', 'pattern': s_config.re_iden},
|
|
@@ -93,6 +94,7 @@ _CronJobSchema = {
|
|
|
93
94
|
'iden': {'type': 'string', 'pattern': s_config.re_iden},
|
|
94
95
|
'view': {'type': 'string', 'pattern': s_config.re_iden},
|
|
95
96
|
'name': {'type': 'string'},
|
|
97
|
+
'pool': {'type': 'boolean'},
|
|
96
98
|
'doc': {'type': 'string'},
|
|
97
99
|
'incunit': {
|
|
98
100
|
'oneOf': [
|
|
@@ -203,6 +205,7 @@ reqValidMerge = s_config.getJsValidator({
|
|
|
203
205
|
'creator': {'type': 'string', 'pattern': s_config.re_iden},
|
|
204
206
|
'created': {'type': 'number', 'minval': 0},
|
|
205
207
|
'comment': {'type': 'string'},
|
|
208
|
+
'updated': {'type': 'number', 'minval': 0},
|
|
206
209
|
},
|
|
207
210
|
'required': ['iden', 'creator', 'created'],
|
|
208
211
|
'additionalProperties': False,
|
|
@@ -216,6 +219,7 @@ reqValidVote = s_config.getJsValidator({
|
|
|
216
219
|
'created': {'type': 'number', 'minval': 0},
|
|
217
220
|
'approved': {'type': 'boolean'},
|
|
218
221
|
'comment': {'type': 'string'},
|
|
222
|
+
'updated': {'type': 'number', 'minval': 0},
|
|
219
223
|
},
|
|
220
224
|
'required': ['user', 'offset', 'created', 'approved'],
|
|
221
225
|
'additionalProperties': False,
|
|
@@ -273,3 +277,13 @@ reqValidSslCtxOpts = s_config.getJsValidator({
|
|
|
273
277
|
},
|
|
274
278
|
'additionalProperties': False,
|
|
275
279
|
})
|
|
280
|
+
|
|
281
|
+
_stormPoolOptsSchema = {
|
|
282
|
+
'type': 'object',
|
|
283
|
+
'properties': {
|
|
284
|
+
'timeout:sync': {'type': 'integer', 'minimum': 1},
|
|
285
|
+
'timeout:connection': {'type': 'integer', 'minimum': 1},
|
|
286
|
+
},
|
|
287
|
+
'additionalProperties': False,
|
|
288
|
+
}
|
|
289
|
+
reqValidStormPoolOpts = s_config.getJsValidator(_stormPoolOptsSchema)
|
synapse/lib/snap.py
CHANGED
|
@@ -209,6 +209,27 @@ class ProtoNode:
|
|
|
209
209
|
if tagnode is not s_common.novalu:
|
|
210
210
|
return self.ctx.loadNode(tagnode)
|
|
211
211
|
|
|
212
|
+
# check for an :isnow tag redirection in our hierarchy...
|
|
213
|
+
toks = info.get('toks')
|
|
214
|
+
for i in range(len(toks)):
|
|
215
|
+
|
|
216
|
+
toktag = '.'.join(toks[:i + 1])
|
|
217
|
+
toknode = await self.ctx.snap.getTagNode(toktag)
|
|
218
|
+
if toknode is s_common.novalu:
|
|
219
|
+
continue
|
|
220
|
+
|
|
221
|
+
tokvalu = toknode.ndef[1]
|
|
222
|
+
if tokvalu == toktag:
|
|
223
|
+
continue
|
|
224
|
+
|
|
225
|
+
realnow = tokvalu + norm[len(toktag):]
|
|
226
|
+
tagnode = await self.ctx.snap.getTagNode(realnow)
|
|
227
|
+
if tagnode is not s_common.novalu:
|
|
228
|
+
return self.ctx.loadNode(tagnode)
|
|
229
|
+
|
|
230
|
+
norm, info = await self.ctx.snap.getTagNorm(realnow)
|
|
231
|
+
break
|
|
232
|
+
|
|
212
233
|
return await self.ctx.addNode('syn:tag', norm, norminfo=info)
|
|
213
234
|
|
|
214
235
|
def getTag(self, tag):
|
|
@@ -619,10 +640,6 @@ class Snap(s_base.Base):
|
|
|
619
640
|
|
|
620
641
|
self.changelog = []
|
|
621
642
|
self.tagtype = self.core.model.type('ival')
|
|
622
|
-
self.trigson = self.core.trigson
|
|
623
|
-
|
|
624
|
-
def disableTriggers(self):
|
|
625
|
-
self.trigson = False
|
|
626
643
|
|
|
627
644
|
async def getSnapMeta(self):
|
|
628
645
|
'''
|
|
@@ -661,6 +678,33 @@ class Snap(s_base.Base):
|
|
|
661
678
|
self.onfini(runt)
|
|
662
679
|
return runt
|
|
663
680
|
|
|
681
|
+
async def _joinEmbedStor(self, storage, embeds):
|
|
682
|
+
for nodePath, relProps in embeds.items():
|
|
683
|
+
await asyncio.sleep(0)
|
|
684
|
+
iden = relProps.get('*')
|
|
685
|
+
if not iden:
|
|
686
|
+
continue
|
|
687
|
+
|
|
688
|
+
stor = await self.view.getStorNodes(s_common.uhex(iden))
|
|
689
|
+
for relProp in relProps.keys():
|
|
690
|
+
await asyncio.sleep(0)
|
|
691
|
+
if relProp == '*':
|
|
692
|
+
continue
|
|
693
|
+
|
|
694
|
+
for idx, layrstor in enumerate(stor):
|
|
695
|
+
await asyncio.sleep(0)
|
|
696
|
+
props = layrstor.get('props')
|
|
697
|
+
if not props:
|
|
698
|
+
continue
|
|
699
|
+
|
|
700
|
+
if relProp not in props:
|
|
701
|
+
continue
|
|
702
|
+
|
|
703
|
+
if 'embeds' not in storage[idx]:
|
|
704
|
+
storage[idx]['embeds'] = {}
|
|
705
|
+
|
|
706
|
+
storage[idx]['embeds'][f'{nodePath}::{relProp}'] = props[relProp]
|
|
707
|
+
|
|
664
708
|
async def iterStormPodes(self, text, opts, user=None):
|
|
665
709
|
'''
|
|
666
710
|
Yield packed node tuples for the given storm query text.
|
|
@@ -703,6 +747,8 @@ class Snap(s_base.Base):
|
|
|
703
747
|
embdef = embeds.get(node.form.name)
|
|
704
748
|
if embdef is not None:
|
|
705
749
|
pode[1]['embeds'] = await node.getEmbeds(embdef)
|
|
750
|
+
if show_storage:
|
|
751
|
+
await self._joinEmbedStor(pode[1]['storage'], pode[1]['embeds'])
|
|
706
752
|
|
|
707
753
|
yield pode
|
|
708
754
|
|
synapse/lib/storm.lark
CHANGED
|
@@ -56,8 +56,8 @@ edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS
|
|
|
56
56
|
edittagpropdel: EXPRMINUS tagprop
|
|
57
57
|
EQSPACE: /((?<=\s)=|=(?=\s))/
|
|
58
58
|
MODSET.4: "+=" | "-="
|
|
59
|
-
TRYSETPLUS: "?+="
|
|
60
|
-
TRYSETMINUS: "?-="
|
|
59
|
+
TRYSETPLUS.1: "?+="
|
|
60
|
+
TRYSETMINUS.1: "?-="
|
|
61
61
|
TRYSET.1: "?="
|
|
62
62
|
SETTAGOPER: "?"
|
|
63
63
|
|
|
@@ -308,7 +308,7 @@ evalvalu: _valu
|
|
|
308
308
|
exprdict: "{" ((_exprvalu | VARTOKN) (":" | _EXPRCOLONNOSPACE) (_exprvalu | VARTOKN) ("," (_exprvalu | VARTOKN) (":" | _EXPRCOLONNOSPACE) (_exprvalu | VARTOKN))* ","? )? "}"
|
|
309
309
|
exprlist: "[" ((_exprvalu | VARTOKN) ("," (_exprvalu | VARTOKN))* ","? )? "]"
|
|
310
310
|
// Just like _valu, but doesn't allow valu lists or unquoted strings or queries
|
|
311
|
-
_exprvalu: NUMBER | HEXNUMBER | OCTNUMBER | BOOL | exprlist | exprdict | _exprvarvalu | exprrelpropvalu
|
|
311
|
+
_exprvalu: NUMBER | HEXNUMBER | OCTNUMBER | BOOL | NULL | exprlist | exprdict | _exprvarvalu | exprrelpropvalu
|
|
312
312
|
| exprunivpropvalu | exprtagvalu | exprtagpropvalu | TRIPLEQUOTEDSTRING | DOUBLEQUOTEDSTRING
|
|
313
313
|
| SINGLEQUOTEDSTRING | formatstring | _innerdollarexprs
|
|
314
314
|
|
|
@@ -590,6 +590,7 @@ OCTNUMBER.1: /
|
|
|
590
590
|
/x
|
|
591
591
|
|
|
592
592
|
BOOL.2: /(true|false)(?=$|[\s\),\]}\|\=])/
|
|
593
|
+
NULL.2: "null"
|
|
593
594
|
NOT.2: "not"
|
|
594
595
|
OR.2: "or"
|
|
595
596
|
AND.2: "and"
|
synapse/lib/storm.py
CHANGED
|
@@ -27,8 +27,8 @@ import synapse.lib.msgpack as s_msgpack
|
|
|
27
27
|
import synapse.lib.spooled as s_spooled
|
|
28
28
|
import synapse.lib.version as s_version
|
|
29
29
|
import synapse.lib.hashitem as s_hashitem
|
|
30
|
+
import synapse.lib.hiveauth as s_hiveauth
|
|
30
31
|
import synapse.lib.stormctrl as s_stormctrl
|
|
31
|
-
import synapse.lib.provenance as s_provenance
|
|
32
32
|
import synapse.lib.stormtypes as s_stormtypes
|
|
33
33
|
|
|
34
34
|
import synapse.lib.stormlib.graph as s_stormlib_graph
|
|
@@ -225,6 +225,7 @@ reqValidPkgdef = s_config.getJsValidator({
|
|
|
225
225
|
},
|
|
226
226
|
'required': ['cert', 'sign'],
|
|
227
227
|
},
|
|
228
|
+
# TODO: Remove me after Synapse 3.0.0.
|
|
228
229
|
'synapse_minversion': {
|
|
229
230
|
'type': ['array', 'null'],
|
|
230
231
|
'items': {'type': 'number'}
|
|
@@ -1206,6 +1207,8 @@ stormcmds = (
|
|
|
1206
1207
|
'descr': addcrondescr,
|
|
1207
1208
|
'cmdargs': (
|
|
1208
1209
|
('query', {'help': 'Query for the cron job to execute.'}),
|
|
1210
|
+
('--pool', {'action': 'store_true', 'default': False,
|
|
1211
|
+
'help': 'Allow the cron job to be run by a mirror from the query pool.'}),
|
|
1209
1212
|
('--minute', {'help': 'Minute value for job or recurrence period.'}),
|
|
1210
1213
|
('--name', {'help': 'An optional name for the cron job.'}),
|
|
1211
1214
|
('--doc', {'help': 'An optional doc string for the cron job.'}),
|
|
@@ -1225,6 +1228,7 @@ stormcmds = (
|
|
|
1225
1228
|
minute=$cmdopts.minute,
|
|
1226
1229
|
hour=$cmdopts.hour,
|
|
1227
1230
|
day=$cmdopts.day,
|
|
1231
|
+
pool=$cmdopts.pool,
|
|
1228
1232
|
month=$cmdopts.month,
|
|
1229
1233
|
year=$cmdopts.year,
|
|
1230
1234
|
hourly=$cmdopts.hourly,
|
|
@@ -1372,6 +1376,7 @@ stormcmds = (
|
|
|
1372
1376
|
$lib.print('iden: {iden}', iden=$job.iden)
|
|
1373
1377
|
$lib.print('user: {user}', user=$job.user)
|
|
1374
1378
|
$lib.print('enabled: {enabled}', enabled=$job.enabled)
|
|
1379
|
+
$lib.print(`pool: {$job.pool}`)
|
|
1375
1380
|
$lib.print('recurring: {isrecur}', isrecur=$job.isrecur)
|
|
1376
1381
|
$lib.print('# starts: {startcount}', startcount=$job.startcount)
|
|
1377
1382
|
$lib.print('# errors: {errcount}', errcount=$job.errcount)
|
|
@@ -1824,6 +1829,8 @@ class Runtime(s_base.Base):
|
|
|
1824
1829
|
opaque object which is called through, but not dereferenced.
|
|
1825
1830
|
|
|
1826
1831
|
'''
|
|
1832
|
+
|
|
1833
|
+
_admin_reason = s_hiveauth._allowedReason(True, isadmin=True)
|
|
1827
1834
|
async def __anit__(self, query, snap, opts=None, user=None, root=None):
|
|
1828
1835
|
|
|
1829
1836
|
await s_base.Base.__anit__(self)
|
|
@@ -2187,25 +2194,93 @@ class Runtime(s_base.Base):
|
|
|
2187
2194
|
|
|
2188
2195
|
return self.user.allowed(perms, gateiden=gateiden, default=default)
|
|
2189
2196
|
|
|
2197
|
+
def allowedReason(self, perms, gateiden=None, default=None):
|
|
2198
|
+
'''
|
|
2199
|
+
Similar to allowed, but always prefer the default value specified by the caller.
|
|
2200
|
+
Default values are still pulled from permdefs if there is a match there; but still prefer caller default.
|
|
2201
|
+
This results in a ternary response that can be used to know if a rule had a positive/negative or no match.
|
|
2202
|
+
The matching reason metadata is also returned.
|
|
2203
|
+
'''
|
|
2204
|
+
if self.asroot:
|
|
2205
|
+
return self._admin_reason
|
|
2206
|
+
|
|
2207
|
+
if default is None:
|
|
2208
|
+
permdef = self.snap.core.getPermDef(perms)
|
|
2209
|
+
if permdef:
|
|
2210
|
+
default = permdef.get('default', default)
|
|
2211
|
+
|
|
2212
|
+
return self.user.getAllowedReason(perms, gateiden=gateiden, default=default)
|
|
2213
|
+
|
|
2190
2214
|
def confirmPropSet(self, prop, layriden=None):
|
|
2191
2215
|
|
|
2192
2216
|
if layriden is None:
|
|
2193
2217
|
layriden = self.snap.wlyr.iden
|
|
2194
2218
|
|
|
2195
|
-
|
|
2219
|
+
meta0 = self.allowedReason(prop.setperms[0], gateiden=layriden)
|
|
2220
|
+
|
|
2221
|
+
if meta0.isadmin:
|
|
2196
2222
|
return
|
|
2197
2223
|
|
|
2198
|
-
|
|
2224
|
+
allowed0 = meta0.value
|
|
2225
|
+
|
|
2226
|
+
meta1 = self.allowedReason(prop.setperms[1], gateiden=layriden)
|
|
2227
|
+
allowed1 = meta1.value
|
|
2228
|
+
|
|
2229
|
+
if allowed0:
|
|
2230
|
+
if allowed1:
|
|
2231
|
+
return
|
|
2232
|
+
elif allowed1 is False:
|
|
2233
|
+
# This is a allow-with-precedence case.
|
|
2234
|
+
# Inspect meta to determine if the rule a0 is more specific than rule a1
|
|
2235
|
+
if len(meta0.rule) >= len(meta1.rule):
|
|
2236
|
+
return
|
|
2237
|
+
self.user.raisePermDeny(prop.setperms[0], gateiden=layriden)
|
|
2238
|
+
return
|
|
2239
|
+
|
|
2240
|
+
if allowed1:
|
|
2241
|
+
if allowed0 is None:
|
|
2242
|
+
return
|
|
2243
|
+
# allowed0 here is False. This is a deny-with-precedence case.
|
|
2244
|
+
# Inspect meta to determine if the rule a1 is more specific than rule a0
|
|
2245
|
+
if len(meta1.rule) > len(meta0.rule):
|
|
2246
|
+
return
|
|
2247
|
+
|
|
2248
|
+
self.user.raisePermDeny(prop.setperms[0], gateiden=layriden)
|
|
2199
2249
|
|
|
2200
2250
|
def confirmPropDel(self, prop, layriden=None):
|
|
2201
2251
|
|
|
2202
2252
|
if layriden is None:
|
|
2203
2253
|
layriden = self.snap.wlyr.iden
|
|
2204
2254
|
|
|
2205
|
-
|
|
2255
|
+
meta0 = self.allowedReason(prop.delperms[0], gateiden=layriden)
|
|
2256
|
+
|
|
2257
|
+
if meta0.isadmin:
|
|
2258
|
+
return
|
|
2259
|
+
|
|
2260
|
+
allowed0 = meta0.value
|
|
2261
|
+
meta1 = self.allowedReason(prop.delperms[1], gateiden=layriden)
|
|
2262
|
+
allowed1 = meta1.value
|
|
2263
|
+
|
|
2264
|
+
if allowed0:
|
|
2265
|
+
if allowed1:
|
|
2266
|
+
return
|
|
2267
|
+
elif allowed1 is False:
|
|
2268
|
+
# This is a allow-with-precedence case.
|
|
2269
|
+
# Inspect meta to determine if the rule a0 is more specific than rule a1
|
|
2270
|
+
if len(meta0.rule) >= len(meta1.rule):
|
|
2271
|
+
return
|
|
2272
|
+
self.user.raisePermDeny(prop.delperms[0], gateiden=layriden)
|
|
2206
2273
|
return
|
|
2207
2274
|
|
|
2208
|
-
|
|
2275
|
+
if allowed1:
|
|
2276
|
+
if allowed0 is None:
|
|
2277
|
+
return
|
|
2278
|
+
# allowed0 here is False. This is a deny-with-precedence case.
|
|
2279
|
+
# Inspect meta to determine if the rule a1 is more specific than rule a0
|
|
2280
|
+
if len(meta1.rule) > len(meta0.rule):
|
|
2281
|
+
return
|
|
2282
|
+
|
|
2283
|
+
self.user.raisePermDeny(prop.delperms[0], gateiden=layriden)
|
|
2209
2284
|
|
|
2210
2285
|
def confirmEasyPerm(self, item, perm, mesg=None):
|
|
2211
2286
|
if not self.asroot:
|
|
@@ -2238,27 +2313,26 @@ class Runtime(s_base.Base):
|
|
|
2238
2313
|
|
|
2239
2314
|
async def execute(self, genr=None):
|
|
2240
2315
|
try:
|
|
2241
|
-
with s_provenance.claim('storm', q=self.query.text, user=self.user.iden):
|
|
2242
2316
|
|
|
2243
|
-
|
|
2244
|
-
|
|
2317
|
+
async with contextlib.aclosing(self.query.iterNodePaths(self, genr=genr)) as nodegenr:
|
|
2318
|
+
nodegenr, empty = await s_ast.pullone(nodegenr)
|
|
2245
2319
|
|
|
2246
|
-
|
|
2247
|
-
|
|
2320
|
+
if empty:
|
|
2321
|
+
return
|
|
2248
2322
|
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2323
|
+
rules = self.opts.get('graph')
|
|
2324
|
+
if rules not in (False, None):
|
|
2325
|
+
if rules is True:
|
|
2326
|
+
rules = {'degrees': None, 'refs': True}
|
|
2327
|
+
elif isinstance(rules, str):
|
|
2328
|
+
rules = await self.snap.core.getStormGraph(rules)
|
|
2255
2329
|
|
|
2256
|
-
|
|
2257
|
-
|
|
2330
|
+
subgraph = s_ast.SubGraph(rules)
|
|
2331
|
+
nodegenr = subgraph.run(self, nodegenr)
|
|
2258
2332
|
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2333
|
+
async for item in nodegenr:
|
|
2334
|
+
self.tick()
|
|
2335
|
+
yield item
|
|
2262
2336
|
|
|
2263
2337
|
except RecursionError:
|
|
2264
2338
|
mesg = 'Maximum Storm pipeline depth exceeded.'
|
|
@@ -3589,7 +3663,7 @@ class CopyToCmd(Cmd):
|
|
|
3589
3663
|
|
|
3590
3664
|
runt.confirm(node.form.addperm, gateiden=layriden)
|
|
3591
3665
|
for name in node.props.keys():
|
|
3592
|
-
runt.confirmPropSet(node.form.props[name])
|
|
3666
|
+
runt.confirmPropSet(node.form.props[name], layriden=layriden)
|
|
3593
3667
|
|
|
3594
3668
|
for tag in node.tags.keys():
|
|
3595
3669
|
runt.confirm(('node', 'tag', 'add', *tag.split('.')), gateiden=layriden)
|
synapse/lib/storm_format.py
CHANGED
synapse/lib/stormlib/aha.py
CHANGED
|
@@ -35,6 +35,7 @@ class AhaPoolLib(s_stormtypes.Lib):
|
|
|
35
35
|
if poolinfo is not None:
|
|
36
36
|
return AhaPool(self.runt, poolinfo)
|
|
37
37
|
|
|
38
|
+
@s_stormtypes.stormfunc(readonly=True)
|
|
38
39
|
async def list(self):
|
|
39
40
|
|
|
40
41
|
self.runt.reqAdmin()
|
|
@@ -45,6 +46,9 @@ class AhaPoolLib(s_stormtypes.Lib):
|
|
|
45
46
|
|
|
46
47
|
@s_stormtypes.registry.registerType
|
|
47
48
|
class AhaPool(s_stormtypes.StormType):
|
|
49
|
+
'''
|
|
50
|
+
Implements the Storm API for an AHA pool.
|
|
51
|
+
'''
|
|
48
52
|
_storm_typename = 'aha:pool'
|
|
49
53
|
|
|
50
54
|
def __init__(self, runt, poolinfo):
|
|
@@ -57,6 +61,9 @@ class AhaPool(s_stormtypes.StormType):
|
|
|
57
61
|
'del': self._del,
|
|
58
62
|
})
|
|
59
63
|
|
|
64
|
+
async def stormrepr(self):
|
|
65
|
+
return f'{self._storm_typename}: {self.poolinfo.get("name")}'
|
|
66
|
+
|
|
60
67
|
async def _derefGet(self, name):
|
|
61
68
|
return self.poolinfo.get(name)
|
|
62
69
|
|
|
@@ -94,7 +101,6 @@ stormcmds = (
|
|
|
94
101
|
for $pool in $lib.aha.pool.list() {
|
|
95
102
|
$count = ($count + 1)
|
|
96
103
|
$lib.print(`Pool: {$pool.name}`)
|
|
97
|
-
$lib.print($pool)
|
|
98
104
|
for ($svcname, $svcinfo) in $pool.services {
|
|
99
105
|
$lib.print(` {$svcname}`)
|
|
100
106
|
}
|
synapse/lib/stormlib/auth.py
CHANGED
|
@@ -1192,7 +1192,8 @@ class User(s_stormtypes.Prim):
|
|
|
1192
1192
|
|
|
1193
1193
|
perm = tuple(permname.split('.'))
|
|
1194
1194
|
user = await self.runt.snap.core.auth.reqUser(self.valu)
|
|
1195
|
-
|
|
1195
|
+
reason = user.getAllowedReason(perm, gateiden=gateiden, default=default)
|
|
1196
|
+
return reason.value, reason.mesg
|
|
1196
1197
|
|
|
1197
1198
|
async def _methUserGrant(self, iden, indx=None):
|
|
1198
1199
|
self.runt.confirm(('auth', 'user', 'grant'))
|
|
@@ -1226,6 +1227,11 @@ class User(s_stormtypes.Prim):
|
|
|
1226
1227
|
indx = await s_stormtypes.toint(indx, noneok=True)
|
|
1227
1228
|
gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
|
|
1228
1229
|
self.runt.confirm(('auth', 'user', 'set', 'rules'), gateiden=gateiden)
|
|
1230
|
+
# TODO: Remove me in 3.0.0
|
|
1231
|
+
if gateiden == 'cortex':
|
|
1232
|
+
mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
|
|
1233
|
+
f'for permission checks and will be removed in Synapse v3.0.0.'
|
|
1234
|
+
await self.runt.snap.warn(mesg, log=False)
|
|
1229
1235
|
await self.runt.snap.core.addUserRule(self.valu, rule, indx=indx, gateiden=gateiden)
|
|
1230
1236
|
|
|
1231
1237
|
async def _methUserDelRule(self, rule, gateiden=None):
|
|
@@ -1479,6 +1485,11 @@ class Role(s_stormtypes.Prim):
|
|
|
1479
1485
|
indx = await s_stormtypes.toint(indx, noneok=True)
|
|
1480
1486
|
gateiden = await s_stormtypes.tostr(gateiden, noneok=True)
|
|
1481
1487
|
self.runt.confirm(('auth', 'role', 'set', 'rules'), gateiden=gateiden)
|
|
1488
|
+
# TODO: Remove me in 3.0.0
|
|
1489
|
+
if gateiden == 'cortex':
|
|
1490
|
+
mesg = f'Adding rule on the "cortex" authgate. This authgate is not used ' \
|
|
1491
|
+
f'for permission checks and will be removed in Synapse v3.0.0.'
|
|
1492
|
+
await self.runt.snap.warn(mesg, log=False)
|
|
1482
1493
|
await self.runt.snap.core.addRoleRule(self.valu, rule, indx=indx, gateiden=gateiden)
|
|
1483
1494
|
|
|
1484
1495
|
async def _methRoleDelRule(self, rule, gateiden=None):
|
|
@@ -1557,10 +1568,7 @@ class LibAuth(s_stormtypes.Lib):
|
|
|
1557
1568
|
@s_stormtypes.stormfunc(readonly=True)
|
|
1558
1569
|
async def textFromRule(self, rule):
|
|
1559
1570
|
rule = await s_stormtypes.toprim(rule)
|
|
1560
|
-
|
|
1561
|
-
if not rule[0]:
|
|
1562
|
-
text = '!' + text
|
|
1563
|
-
return text
|
|
1571
|
+
return s_common.reprauthrule(rule)
|
|
1564
1572
|
|
|
1565
1573
|
@s_stormtypes.stormfunc(readonly=True)
|
|
1566
1574
|
async def getPermDefs(self):
|