synapse 2.222.0__py311-none-any.whl → 2.224.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/common.py +2 -2
- synapse/cortex.py +159 -31
- synapse/cryotank.py +1 -1
- synapse/datamodel.py +1 -1
- synapse/lib/ast.py +5 -3
- synapse/lib/layer.py +6 -6
- synapse/lib/nexus.py +1 -1
- synapse/lib/schemas.py +2 -0
- synapse/lib/snap.py +15 -9
- synapse/lib/storm.py +35 -191
- synapse/lib/stormlib/auth.py +1 -1
- synapse/lib/stormlib/imap.py +12 -4
- synapse/lib/stormlib/mime.py +15 -5
- synapse/lib/stormlib/pkg.py +598 -0
- synapse/lib/stormlib/task.py +114 -0
- synapse/lib/stormtypes.py +43 -175
- synapse/lib/trigger.py +16 -14
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +17 -14
- synapse/models/files.py +1 -1
- synapse/models/inet.py +25 -0
- synapse/models/proj.py +3 -0
- synapse/models/risk.py +6 -0
- synapse/models/syn.py +8 -0
- synapse/tests/test_common.py +4 -0
- synapse/tests/test_cortex.py +52 -1
- synapse/tests/test_lib_aha.py +68 -53
- synapse/tests/test_lib_ast.py +3 -0
- synapse/tests/test_lib_cell.py +12 -12
- synapse/tests/test_lib_storm.py +128 -248
- synapse/tests/test_lib_stormlib_imap.py +14 -0
- synapse/tests/test_lib_stormlib_mime.py +24 -0
- synapse/tests/test_lib_stormlib_pkg.py +456 -0
- synapse/tests/test_lib_stormlib_task.py +98 -0
- synapse/tests/test_lib_stormtypes.py +12 -100
- synapse/tests/test_lib_trigger.py +66 -3
- synapse/tests/test_lib_view.py +53 -0
- synapse/tests/test_model_files.py +11 -0
- synapse/tests/test_model_inet.py +23 -0
- synapse/tests/test_model_proj.py +3 -1
- synapse/tests/test_model_risk.py +10 -0
- synapse/tests/test_model_syn.py +54 -2
- synapse/tools/cryo/cat.py +2 -1
- synapse/tools/cryo/list.py +2 -0
- {synapse-2.222.0.dist-info → synapse-2.224.0.dist-info}/METADATA +1 -1
- {synapse-2.222.0.dist-info → synapse-2.224.0.dist-info}/RECORD +49 -45
- {synapse-2.222.0.dist-info → synapse-2.224.0.dist-info}/WHEEL +0 -0
- {synapse-2.222.0.dist-info → synapse-2.224.0.dist-info}/licenses/LICENSE +0 -0
- {synapse-2.222.0.dist-info → synapse-2.224.0.dist-info}/top_level.txt +0 -0
synapse/common.py
CHANGED
|
@@ -144,7 +144,7 @@ def flatten(item):
|
|
|
144
144
|
item: The python primitive object to normalize.
|
|
145
145
|
|
|
146
146
|
Notes:
|
|
147
|
-
Only None, bool, int, bytes, strings, lists, tuples and dictionaries are acceptable input.
|
|
147
|
+
Only None, bool, int, bytes, strings, floats, lists, tuples and dictionaries are acceptable input.
|
|
148
148
|
List objects will be converted to tuples.
|
|
149
149
|
Dictionary objects must have keys which can be sorted.
|
|
150
150
|
|
|
@@ -155,7 +155,7 @@ def flatten(item):
|
|
|
155
155
|
if item is None:
|
|
156
156
|
return None
|
|
157
157
|
|
|
158
|
-
if isinstance(item, (str, int, bytes)):
|
|
158
|
+
if isinstance(item, (str, int, bytes, float)):
|
|
159
159
|
return item
|
|
160
160
|
|
|
161
161
|
if isinstance(item, (tuple, list)):
|
synapse/cortex.py
CHANGED
|
@@ -61,6 +61,7 @@ import synapse.lib.stormlib.gen as s_stormlib_gen # NOQA
|
|
|
61
61
|
import synapse.lib.stormlib.gis as s_stormlib_gis # NOQA
|
|
62
62
|
import synapse.lib.stormlib.hex as s_stormlib_hex # NOQA
|
|
63
63
|
import synapse.lib.stormlib.log as s_stormlib_log # NOQA
|
|
64
|
+
import synapse.lib.stormlib.pkg as s_stormlib_pkg # NOQA
|
|
64
65
|
import synapse.lib.stormlib.xml as s_stormlib_xml # NOQA
|
|
65
66
|
import synapse.lib.stormlib.auth as s_stormlib_auth # NOQA
|
|
66
67
|
import synapse.lib.stormlib.cell as s_stormlib_cell # NOQA
|
|
@@ -72,6 +73,7 @@ import synapse.lib.stormlib.mime as s_stormlib_mime # NOQA
|
|
|
72
73
|
import synapse.lib.stormlib.pack as s_stormlib_pack # NOQA
|
|
73
74
|
import synapse.lib.stormlib.smtp as s_stormlib_smtp # NOQA
|
|
74
75
|
import synapse.lib.stormlib.stix as s_stormlib_stix # NOQA
|
|
76
|
+
import synapse.lib.stormlib.task as s_stormlib_task # NOQA
|
|
75
77
|
import synapse.lib.stormlib.yaml as s_stormlib_yaml # NOQA
|
|
76
78
|
import synapse.lib.stormlib.basex as s_stormlib_basex # NOQA
|
|
77
79
|
import synapse.lib.stormlib.cache as s_stormlib_cache # NOQA
|
|
@@ -462,6 +464,15 @@ class CoreApi(s_cell.CellApi):
|
|
|
462
464
|
'''
|
|
463
465
|
return await self.cell.getTypeNorm(name, valu, typeopts=typeopts)
|
|
464
466
|
|
|
467
|
+
async def addType(self, typename, basetype, typeopts, typeinfo):
|
|
468
|
+
'''
|
|
469
|
+
Add an extended type to the data model.
|
|
470
|
+
|
|
471
|
+
Extended types must begin with _
|
|
472
|
+
'''
|
|
473
|
+
self.user.confirm(('model', 'type', 'add', typename))
|
|
474
|
+
return await self.cell.addType(typename, basetype, typeopts, typeinfo)
|
|
475
|
+
|
|
465
476
|
async def addForm(self, formname, basetype, typeopts, typeinfo):
|
|
466
477
|
'''
|
|
467
478
|
Add an extended form to the data model.
|
|
@@ -533,6 +544,15 @@ class CoreApi(s_cell.CellApi):
|
|
|
533
544
|
self.user.confirm(('model', 'tagprop', 'del'))
|
|
534
545
|
return await self.cell.delTagProp(name)
|
|
535
546
|
|
|
547
|
+
async def addEdge(self, edge, edgeinfo):
|
|
548
|
+
'''
|
|
549
|
+
Add an extended edge definition to the data model.
|
|
550
|
+
|
|
551
|
+
Extended edge definitions must use a verb which begins with _
|
|
552
|
+
'''
|
|
553
|
+
self.user.confirm(('model', 'edge', 'add'))
|
|
554
|
+
return await self.cell.addEdge(edge, edgeinfo)
|
|
555
|
+
|
|
536
556
|
async def addStormPkg(self, pkgdef, verify=False):
|
|
537
557
|
self.user.confirm(('pkg', 'add'))
|
|
538
558
|
return await self.cell.addStormPkg(pkgdef, verify=verify)
|
|
@@ -1438,6 +1458,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1438
1458
|
{'perm': ('model', 'univ', 'del'), 'gate': 'cortex',
|
|
1439
1459
|
'desc': 'Controls access to deleting extended model universal properties and values.'},
|
|
1440
1460
|
|
|
1461
|
+
{'perm': ('model', 'edge', 'add'), 'gate': 'cortex',
|
|
1462
|
+
'desc': 'Controls access to adding extended model edges.'},
|
|
1463
|
+
{'perm': ('model', 'edge', 'del'), 'gate': 'cortex',
|
|
1464
|
+
'desc': 'Controls access to deleting extended model edges.'},
|
|
1465
|
+
|
|
1441
1466
|
{'perm': ('node',), 'gate': 'layer',
|
|
1442
1467
|
'desc': 'Controls all node edits in a layer.'},
|
|
1443
1468
|
{'perm': ('node', 'add'), 'gate': 'layer',
|
|
@@ -2262,6 +2287,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
2262
2287
|
self.onfini(slab.fini)
|
|
2263
2288
|
|
|
2264
2289
|
self.multiqueue = await slab.getMultiQueue('cortex:queue', nexsroot=self.nexsroot)
|
|
2290
|
+
self.stormpkgqueue = await slab.getMultiQueue('storm:pkg:queue', nexsroot=self.nexsroot)
|
|
2265
2291
|
|
|
2266
2292
|
async def _initStormGraphs(self):
|
|
2267
2293
|
path = os.path.join(self.dirn, 'slabs', 'graphs.lmdb')
|
|
@@ -2565,6 +2591,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
2565
2591
|
ctor.pkgname = cdef.get('pkgname')
|
|
2566
2592
|
ctor.svciden = cdef.get('cmdconf', {}).get('svciden', '')
|
|
2567
2593
|
ctor.forms = cdef.get('forms', {})
|
|
2594
|
+
ctor.deprecated = cdef.get('deprecated', {})
|
|
2568
2595
|
|
|
2569
2596
|
def getStorNode(form):
|
|
2570
2597
|
ndef = (form.name, form.type.norm(cdef.get('name'))[0])
|
|
@@ -2593,6 +2620,18 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
2593
2620
|
if ctor.pkgname:
|
|
2594
2621
|
props['package'] = ctor.pkgname
|
|
2595
2622
|
|
|
2623
|
+
if ctor.deprecated:
|
|
2624
|
+
props['deprecated'] = True
|
|
2625
|
+
|
|
2626
|
+
if (eolvers := ctor.deprecated.get('eolvers')) is not None:
|
|
2627
|
+
props['deprecated:version'] = eolvers
|
|
2628
|
+
|
|
2629
|
+
if (eoldate := ctor.deprecated.get('eoldate')) is not None:
|
|
2630
|
+
props['deprecated:date'] = eoldate
|
|
2631
|
+
|
|
2632
|
+
if (mesg := ctor.deprecated.get('mesg')) is not None:
|
|
2633
|
+
props['deprecated:mesg'] = mesg
|
|
2634
|
+
|
|
2596
2635
|
pnorms = {}
|
|
2597
2636
|
for prop, valu in props.items():
|
|
2598
2637
|
formprop = form.props.get(prop)
|
|
@@ -3062,8 +3101,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3062
3101
|
if not ok:
|
|
3063
3102
|
break
|
|
3064
3103
|
|
|
3065
|
-
curvers = vers
|
|
3066
|
-
await self.setStormPkgVar(name, varname,
|
|
3104
|
+
curvers = max(vers, await self.getStormPkgVar(name, varname, default=-1))
|
|
3105
|
+
await self.setStormPkgVar(name, varname, curvers)
|
|
3067
3106
|
logger.info(f'{name} finished init vers={vers}: {vname}', extra=logextra)
|
|
3068
3107
|
|
|
3069
3108
|
if onload is not None:
|
|
@@ -3348,6 +3387,91 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3348
3387
|
for item in pkgvars.items():
|
|
3349
3388
|
yield item
|
|
3350
3389
|
|
|
3390
|
+
async def addStormPkgQueue(self, pkgname, name):
|
|
3391
|
+
guid = s_common.guid((pkgname, name))
|
|
3392
|
+
if self.stormpkgqueue.exists(guid):
|
|
3393
|
+
mesg = f'Queue named {name} already exists for package {pkgname}!'
|
|
3394
|
+
raise s_exc.DupName(mesg=mesg)
|
|
3395
|
+
|
|
3396
|
+
info = {
|
|
3397
|
+
'iden': guid,
|
|
3398
|
+
'name': name,
|
|
3399
|
+
'pkgname': pkgname,
|
|
3400
|
+
'created': s_common.now()
|
|
3401
|
+
}
|
|
3402
|
+
|
|
3403
|
+
await self._push('storm:pkg:queue:add', pkgname, name, info)
|
|
3404
|
+
|
|
3405
|
+
@s_nexus.Pusher.onPush('storm:pkg:queue:add')
|
|
3406
|
+
async def _addStormPkgQueue(self, pkgname, name, info):
|
|
3407
|
+
guid = s_common.guid((pkgname, name))
|
|
3408
|
+
if self.stormpkgqueue.exists(guid):
|
|
3409
|
+
return
|
|
3410
|
+
await self.stormpkgqueue.add(guid, info)
|
|
3411
|
+
|
|
3412
|
+
async def listStormPkgQueues(self, pkgname=None):
|
|
3413
|
+
for pkginfo in self.stormpkgqueue.list():
|
|
3414
|
+
if pkgname is None or pkginfo['meta'].get('pkgname') == pkgname:
|
|
3415
|
+
yield pkginfo
|
|
3416
|
+
|
|
3417
|
+
async def getStormPkgQueue(self, pkgname, name):
|
|
3418
|
+
guid = s_common.guid((pkgname, name))
|
|
3419
|
+
return self.stormpkgqueue.status(guid)
|
|
3420
|
+
|
|
3421
|
+
async def delStormPkgQueue(self, pkgname, name):
|
|
3422
|
+
guid = s_common.guid((pkgname, name))
|
|
3423
|
+
if not self.stormpkgqueue.exists(guid):
|
|
3424
|
+
mesg = f'No queue named {name} exists for package {pkgname}!'
|
|
3425
|
+
raise s_exc.NoSuchName(mesg=mesg)
|
|
3426
|
+
|
|
3427
|
+
await self._push('storm:pkg:queue:del', pkgname, name)
|
|
3428
|
+
|
|
3429
|
+
@s_nexus.Pusher.onPush('storm:pkg:queue:del')
|
|
3430
|
+
async def _delStormPkgQueue(self, pkgname, name):
|
|
3431
|
+
guid = s_common.guid((pkgname, name))
|
|
3432
|
+
if not self.stormpkgqueue.exists(guid):
|
|
3433
|
+
return
|
|
3434
|
+
await self.stormpkgqueue.rem(guid)
|
|
3435
|
+
|
|
3436
|
+
async def stormPkgQueueGet(self, pkgname, name, offs=0, wait=False):
|
|
3437
|
+
guid = s_common.guid((pkgname, name))
|
|
3438
|
+
async for item in self.stormpkgqueue.gets(guid, offs, cull=False, wait=wait):
|
|
3439
|
+
return item
|
|
3440
|
+
|
|
3441
|
+
async def stormPkgQueueGets(self, pkgname, name, offs=0, wait=False, size=None):
|
|
3442
|
+
count = 0
|
|
3443
|
+
guid = s_common.guid((pkgname, name))
|
|
3444
|
+
async for item in self.stormpkgqueue.gets(guid, offs, cull=False, wait=wait):
|
|
3445
|
+
|
|
3446
|
+
yield item
|
|
3447
|
+
|
|
3448
|
+
count += 1
|
|
3449
|
+
if size is not None and count >= size:
|
|
3450
|
+
return
|
|
3451
|
+
|
|
3452
|
+
async def stormPkgQueuePuts(self, pkgname, name, items):
|
|
3453
|
+
return await self._push('storm:pkg:queue:puts', pkgname, name, items)
|
|
3454
|
+
|
|
3455
|
+
@s_nexus.Pusher.onPush('storm:pkg:queue:puts', passitem=True)
|
|
3456
|
+
async def _stormPkgQueuePuts(self, pkgname, name, items, nexsitem):
|
|
3457
|
+
nexsoff, nexsmesg = nexsitem
|
|
3458
|
+
guid = s_common.guid((pkgname, name))
|
|
3459
|
+
return await self.stormpkgqueue.puts(guid, items, reqid=nexsoff)
|
|
3460
|
+
|
|
3461
|
+
@s_nexus.Pusher.onPushAuto('storm:pkg:queue:cull')
|
|
3462
|
+
async def stormPkgQueueCull(self, pkgname, name, offs):
|
|
3463
|
+
guid = s_common.guid((pkgname, name))
|
|
3464
|
+
await self.stormpkgqueue.cull(guid, offs)
|
|
3465
|
+
|
|
3466
|
+
@s_nexus.Pusher.onPushAuto('storm:pkg:queue:pop')
|
|
3467
|
+
async def stormPkgQueuePop(self, pkgname, name, offs):
|
|
3468
|
+
guid = s_common.guid((pkgname, name))
|
|
3469
|
+
return await self.stormpkgqueue.pop(guid, offs)
|
|
3470
|
+
|
|
3471
|
+
async def stormPkgQueueSize(self, pkgname, name):
|
|
3472
|
+
guid = s_common.guid((pkgname, name))
|
|
3473
|
+
return self.stormpkgqueue.size(guid)
|
|
3474
|
+
|
|
3351
3475
|
async def _cortexHealth(self, health):
|
|
3352
3476
|
health.update('cortex', 'nominal')
|
|
3353
3477
|
|
|
@@ -3574,6 +3698,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3574
3698
|
|
|
3575
3699
|
return True
|
|
3576
3700
|
|
|
3701
|
+
@s_cell.from_leader
|
|
3577
3702
|
async def addUnivProp(self, name, tdef, info):
|
|
3578
3703
|
if not isinstance(tdef, tuple):
|
|
3579
3704
|
mesg = 'Universal property type definitions should be a tuple.'
|
|
@@ -3588,6 +3713,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3588
3713
|
mesg = 'ext univ name must start with "_"'
|
|
3589
3714
|
raise s_exc.BadPropDef(name=name, mesg=mesg)
|
|
3590
3715
|
|
|
3716
|
+
self.model.getTypeClone(tdef)
|
|
3717
|
+
|
|
3591
3718
|
base = '.' + name
|
|
3592
3719
|
if base in self.model.props:
|
|
3593
3720
|
raise s_exc.DupPropName(mesg=f'Cannot add duplicate universal property {base}',
|
|
@@ -3609,6 +3736,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3609
3736
|
if univ:
|
|
3610
3737
|
await self.feedBeholder('model:univ:add', univ.pack())
|
|
3611
3738
|
|
|
3739
|
+
@s_cell.from_leader
|
|
3612
3740
|
async def addForm(self, formname, basetype, typeopts, typeinfo):
|
|
3613
3741
|
if not isinstance(typeopts, dict):
|
|
3614
3742
|
mesg = 'Form type options should be a dict.'
|
|
@@ -3630,6 +3758,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3630
3758
|
mesg = f'Type already exists: {formname}'
|
|
3631
3759
|
raise s_exc.DupTypeName.init(formname)
|
|
3632
3760
|
|
|
3761
|
+
self.model.getTypeClone((basetype, typeopts))
|
|
3762
|
+
|
|
3633
3763
|
return await self._push('model:form:add', formname, basetype, typeopts, typeinfo)
|
|
3634
3764
|
|
|
3635
3765
|
@s_nexus.Pusher.onPush('model:form:add')
|
|
@@ -3684,6 +3814,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3684
3814
|
await self.fire('core:extmodel:change', form=formname, act='del', type='form')
|
|
3685
3815
|
await self.feedBeholder('model:form:del', {'form': formname})
|
|
3686
3816
|
|
|
3817
|
+
@s_cell.from_leader
|
|
3687
3818
|
async def addType(self, typename, basetype, typeopts, typeinfo):
|
|
3688
3819
|
if not isinstance(typeopts, dict):
|
|
3689
3820
|
mesg = 'Type options should be a dict.'
|
|
@@ -3752,6 +3883,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3752
3883
|
await self.fire('core:extmodel:change', name=typename, act='del', type='type')
|
|
3753
3884
|
await self.feedBeholder('model:type:del', {'type': typename})
|
|
3754
3885
|
|
|
3886
|
+
@s_cell.from_leader
|
|
3755
3887
|
async def addFormProp(self, form, prop, tdef, info):
|
|
3756
3888
|
if not isinstance(tdef, tuple):
|
|
3757
3889
|
mesg = 'Form property type definitions should be a tuple.'
|
|
@@ -3770,6 +3902,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3770
3902
|
if _form.prop(prop):
|
|
3771
3903
|
raise s_exc.DupPropName(mesg=f'Cannot add duplicate form prop {form} {prop}',
|
|
3772
3904
|
form=form, prop=prop)
|
|
3905
|
+
|
|
3906
|
+
self.model.getTypeClone(tdef)
|
|
3907
|
+
|
|
3773
3908
|
await self._push('model:prop:add', form, prop, tdef, info)
|
|
3774
3909
|
|
|
3775
3910
|
@s_nexus.Pusher.onPush('model:prop:add')
|
|
@@ -3951,6 +4086,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3951
4086
|
await self.fire('core:extmodel:change', name=prop, act='del', type='univ')
|
|
3952
4087
|
await self.feedBeholder('model:univ:del', {'prop': univname})
|
|
3953
4088
|
|
|
4089
|
+
@s_cell.from_leader
|
|
3954
4090
|
async def addTagProp(self, name, tdef, info):
|
|
3955
4091
|
if not isinstance(tdef, tuple):
|
|
3956
4092
|
mesg = 'Tag property type definitions should be a tuple.'
|
|
@@ -3963,6 +4099,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3963
4099
|
if self.exttagprops.get(name) is not None:
|
|
3964
4100
|
raise s_exc.DupPropName(name=name)
|
|
3965
4101
|
|
|
4102
|
+
self.model.getTypeClone(tdef)
|
|
4103
|
+
|
|
3966
4104
|
return await self._push('model:tagprop:add', name, tdef, info)
|
|
3967
4105
|
|
|
3968
4106
|
@s_nexus.Pusher.onPush('model:tagprop:add')
|
|
@@ -4000,6 +4138,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4000
4138
|
await self.fire('core:tagprop:change', name=name, act='del')
|
|
4001
4139
|
await self.feedBeholder('model:tagprop:del', {'tagprop': name})
|
|
4002
4140
|
|
|
4141
|
+
@s_cell.from_leader
|
|
4003
4142
|
async def addEdge(self, edge, edgeinfo):
|
|
4004
4143
|
if not isinstance(edgeinfo, dict):
|
|
4005
4144
|
mesg = 'Edge info should be a dict.'
|
|
@@ -4568,35 +4707,24 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4568
4707
|
self.addStormCmd(s_stormlib_cortex.StormPoolGetCmd)
|
|
4569
4708
|
self.addStormCmd(s_stormlib_cortex.StormPoolSetCmd)
|
|
4570
4709
|
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
for
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
for cdef in s_stormlib_model.stormcmds:
|
|
4590
|
-
await self._trySetStormCmd(cdef.get('name'), cdef)
|
|
4591
|
-
|
|
4592
|
-
for cdef in s_stormlib_cortex.stormcmds:
|
|
4593
|
-
await self._trySetStormCmd(cdef.get('name'), cdef)
|
|
4594
|
-
|
|
4595
|
-
for cdef in s_stormlib_vault.stormcmds:
|
|
4596
|
-
await self._trySetStormCmd(cdef.get('name'), cdef)
|
|
4597
|
-
|
|
4598
|
-
for cdef in s_stormlib_index.stormcmds:
|
|
4599
|
-
await self._trySetStormCmd(cdef.get('name'), cdef)
|
|
4710
|
+
cmdmods = [
|
|
4711
|
+
s_storm,
|
|
4712
|
+
s_stormsvc,
|
|
4713
|
+
s_stormlib_aha,
|
|
4714
|
+
s_stormlib_auth,
|
|
4715
|
+
s_stormlib_cortex,
|
|
4716
|
+
s_stormlib_gen,
|
|
4717
|
+
s_stormlib_index,
|
|
4718
|
+
s_stormlib_macro,
|
|
4719
|
+
s_stormlib_model,
|
|
4720
|
+
s_stormlib_pkg,
|
|
4721
|
+
s_stormlib_task,
|
|
4722
|
+
s_stormlib_vault,
|
|
4723
|
+
]
|
|
4724
|
+
|
|
4725
|
+
for cmod in cmdmods:
|
|
4726
|
+
for cdef in cmod.stormcmds:
|
|
4727
|
+
await self._trySetStormCmd(cdef.get('name'), cdef)
|
|
4600
4728
|
|
|
4601
4729
|
async def _initPureStormCmds(self):
|
|
4602
4730
|
oldcmds = []
|
synapse/cryotank.py
CHANGED
|
@@ -39,7 +39,7 @@ class CryoTank(s_base.Base):
|
|
|
39
39
|
A CryoTank implements a stream of structured data.
|
|
40
40
|
'''
|
|
41
41
|
async def __anit__(self, dirn, iden, conf=None):
|
|
42
|
-
|
|
42
|
+
s_common.deprecated('synapse.cryotank.CryoTank', curv='2.223.0')
|
|
43
43
|
await s_base.Base.__anit__(self)
|
|
44
44
|
|
|
45
45
|
if conf is None:
|
synapse/datamodel.py
CHANGED
synapse/lib/ast.py
CHANGED
|
@@ -4527,9 +4527,6 @@ class N1Walk(Oper):
|
|
|
4527
4527
|
|
|
4528
4528
|
def buildfilter(self, runt, destforms, cmpr):
|
|
4529
4529
|
|
|
4530
|
-
if not isinstance(destforms, (tuple, list)):
|
|
4531
|
-
destforms = (destforms,)
|
|
4532
|
-
|
|
4533
4530
|
if '*' in destforms:
|
|
4534
4531
|
if cmpr is not None:
|
|
4535
4532
|
mesg = 'Wild card walk operations do not support comparison.'
|
|
@@ -4614,6 +4611,11 @@ class N1Walk(Oper):
|
|
|
4614
4611
|
dest = await self.kids[1].compute(runt, path)
|
|
4615
4612
|
dest = await s_stormtypes.toprim(dest)
|
|
4616
4613
|
|
|
4614
|
+
if isinstance(dest, (tuple, list)):
|
|
4615
|
+
dest = [await s_stormtypes.tostr(form) for form in dest]
|
|
4616
|
+
else:
|
|
4617
|
+
dest = (await s_stormtypes.tostr(dest),)
|
|
4618
|
+
|
|
4617
4619
|
destfilt = self.buildfilter(runt, dest, cmpr)
|
|
4618
4620
|
|
|
4619
4621
|
for verb in verbs:
|
synapse/lib/layer.py
CHANGED
|
@@ -127,7 +127,7 @@ class LayerApi(s_cell.CellApi):
|
|
|
127
127
|
Scan the full layer and yield artificial nodeedit sets.
|
|
128
128
|
'''
|
|
129
129
|
|
|
130
|
-
if not self.allowed(self.liftperm):
|
|
130
|
+
if not await self.allowed(self.liftperm):
|
|
131
131
|
await self._reqUserAllowed(self.readperm)
|
|
132
132
|
async for item in self.layr.iterLayerNodeEdits():
|
|
133
133
|
yield item
|
|
@@ -167,14 +167,14 @@ class LayerApi(s_cell.CellApi):
|
|
|
167
167
|
|
|
168
168
|
Once caught up with storage, yield them in realtime.
|
|
169
169
|
'''
|
|
170
|
-
if not self.allowed(self.liftperm):
|
|
170
|
+
if not await self.allowed(self.liftperm):
|
|
171
171
|
await self._reqUserAllowed(self.readperm)
|
|
172
172
|
async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
|
|
173
173
|
yield item
|
|
174
174
|
await asyncio.sleep(0)
|
|
175
175
|
|
|
176
176
|
async def syncNodeEdits2(self, offs, wait=True):
|
|
177
|
-
if not self.allowed(self.liftperm):
|
|
177
|
+
if not await self.allowed(self.liftperm):
|
|
178
178
|
await self._reqUserAllowed(self.readperm)
|
|
179
179
|
async for item in self.layr.syncNodeEdits2(offs, wait=wait):
|
|
180
180
|
yield item
|
|
@@ -184,7 +184,7 @@ class LayerApi(s_cell.CellApi):
|
|
|
184
184
|
'''
|
|
185
185
|
Returns what will be the *next* nodeedit log index.
|
|
186
186
|
'''
|
|
187
|
-
if not self.allowed(self.liftperm):
|
|
187
|
+
if not await self.allowed(self.liftperm):
|
|
188
188
|
await self._reqUserAllowed(self.readperm)
|
|
189
189
|
return await self.layr.getEditIndx()
|
|
190
190
|
|
|
@@ -192,12 +192,12 @@ class LayerApi(s_cell.CellApi):
|
|
|
192
192
|
'''
|
|
193
193
|
Return the total number of (edits, meta) pairs in the layer changelog.
|
|
194
194
|
'''
|
|
195
|
-
if not self.allowed(self.liftperm):
|
|
195
|
+
if not await self.allowed(self.liftperm):
|
|
196
196
|
await self._reqUserAllowed(self.readperm)
|
|
197
197
|
return await self.layr.getEditSize()
|
|
198
198
|
|
|
199
199
|
async def getIden(self):
|
|
200
|
-
if not self.allowed(self.liftperm):
|
|
200
|
+
if not await self.allowed(self.liftperm):
|
|
201
201
|
await self._reqUserAllowed(self.readperm)
|
|
202
202
|
return self.layr.iden
|
|
203
203
|
|
synapse/lib/nexus.py
CHANGED
|
@@ -572,7 +572,7 @@ class NexsRoot(s_base.Base):
|
|
|
572
572
|
|
|
573
573
|
offs, args = item
|
|
574
574
|
if offs != self.nexslog.index():
|
|
575
|
-
logger.error('Local Nexus offset is out of sync from remote cell! Aborting mirror sync')
|
|
575
|
+
logger.error(f'Local Nexus offset is out of sync from remote cell! Aborting mirror sync. Local offs={self.nexslog.index()}, Remote {offs=}')
|
|
576
576
|
await self.fini()
|
|
577
577
|
return
|
|
578
578
|
|
synapse/lib/schemas.py
CHANGED
|
@@ -954,6 +954,7 @@ _reqValidPkgdefSchema = {
|
|
|
954
954
|
'items': {'type': 'array',
|
|
955
955
|
'items': {'type': 'string'}},
|
|
956
956
|
},
|
|
957
|
+
'deprecated': {'$ref': '#/definitions/deprecatedItem'},
|
|
957
958
|
},
|
|
958
959
|
'additionalProperties': True,
|
|
959
960
|
'required': ['name', 'storm']
|
|
@@ -980,6 +981,7 @@ _reqValidPkgdefSchema = {
|
|
|
980
981
|
'type': 'string',
|
|
981
982
|
'enum': s_msgpack.deepcopy(datamodel_basetypes),
|
|
982
983
|
},
|
|
984
|
+
'deprecated': {'$ref': '#/definitions/deprecatedItem'},
|
|
983
985
|
},
|
|
984
986
|
}
|
|
985
987
|
],
|
synapse/lib/snap.py
CHANGED
|
@@ -1305,6 +1305,11 @@ class Snap(s_base.Base):
|
|
|
1305
1305
|
mesg = 'The snapshot is in read-only mode.'
|
|
1306
1306
|
raise s_exc.IsReadOnly(mesg=mesg)
|
|
1307
1307
|
|
|
1308
|
+
useriden = meta.get('user')
|
|
1309
|
+
if useriden is None:
|
|
1310
|
+
mesg = 'meta is missing user key. Cannot process edits.'
|
|
1311
|
+
raise s_exc.BadArg(mesg=mesg, name='user')
|
|
1312
|
+
|
|
1308
1313
|
wlyr = self.wlyr
|
|
1309
1314
|
nodes = []
|
|
1310
1315
|
callbacks = []
|
|
@@ -1344,12 +1349,12 @@ class Snap(s_base.Base):
|
|
|
1344
1349
|
if etyp == s_layer.EDIT_NODE_ADD:
|
|
1345
1350
|
node.bylayer['ndef'] = wlyr.iden
|
|
1346
1351
|
callbacks.append((node.form.wasAdded, (node,)))
|
|
1347
|
-
callbacks.append((self.view.runNodeAdd, (node,)))
|
|
1352
|
+
callbacks.append((self.view.runNodeAdd, (node, useriden)))
|
|
1348
1353
|
continue
|
|
1349
1354
|
|
|
1350
1355
|
if etyp == s_layer.EDIT_NODE_DEL:
|
|
1351
1356
|
callbacks.append((node.form.wasDeleted, (node,)))
|
|
1352
|
-
callbacks.append((self.view.runNodeDel, (node,)))
|
|
1357
|
+
callbacks.append((self.view.runNodeDel, (node, useriden)))
|
|
1353
1358
|
continue
|
|
1354
1359
|
|
|
1355
1360
|
if etyp == s_layer.EDIT_PROP_SET:
|
|
@@ -1365,7 +1370,7 @@ class Snap(s_base.Base):
|
|
|
1365
1370
|
node.bylayer['props'][name] = wlyr.iden
|
|
1366
1371
|
|
|
1367
1372
|
callbacks.append((prop.wasSet, (node, oldv)))
|
|
1368
|
-
callbacks.append((self.view.runPropSet, (node, prop, oldv)))
|
|
1373
|
+
callbacks.append((self.view.runPropSet, (node, prop, oldv, useriden)))
|
|
1369
1374
|
continue
|
|
1370
1375
|
|
|
1371
1376
|
if etyp == s_layer.EDIT_PROP_DEL:
|
|
@@ -1381,7 +1386,7 @@ class Snap(s_base.Base):
|
|
|
1381
1386
|
node.bylayer['props'].pop(name, None)
|
|
1382
1387
|
|
|
1383
1388
|
callbacks.append((prop.wasDel, (node, oldv)))
|
|
1384
|
-
callbacks.append((self.view.runPropSet, (node, prop, oldv)))
|
|
1389
|
+
callbacks.append((self.view.runPropSet, (node, prop, oldv, useriden)))
|
|
1385
1390
|
continue
|
|
1386
1391
|
|
|
1387
1392
|
if etyp == s_layer.EDIT_TAG_SET:
|
|
@@ -1391,7 +1396,7 @@ class Snap(s_base.Base):
|
|
|
1391
1396
|
node.tags[tag] = valu
|
|
1392
1397
|
node.bylayer['tags'][tag] = wlyr.iden
|
|
1393
1398
|
|
|
1394
|
-
callbacks.append((self.view.runTagAdd, (node, tag, valu)))
|
|
1399
|
+
callbacks.append((self.view.runTagAdd, (node, tag, valu, useriden,)))
|
|
1395
1400
|
continue
|
|
1396
1401
|
|
|
1397
1402
|
if etyp == s_layer.EDIT_TAG_DEL:
|
|
@@ -1401,7 +1406,7 @@ class Snap(s_base.Base):
|
|
|
1401
1406
|
node.tags.pop(tag, None)
|
|
1402
1407
|
node.bylayer['tags'].pop(tag, None)
|
|
1403
1408
|
|
|
1404
|
-
callbacks.append((self.view.runTagDel, (node, tag, oldv)))
|
|
1409
|
+
callbacks.append((self.view.runTagDel, (node, tag, oldv, useriden)))
|
|
1405
1410
|
continue
|
|
1406
1411
|
|
|
1407
1412
|
if etyp == s_layer.EDIT_TAGPROP_SET:
|
|
@@ -1436,14 +1441,15 @@ class Snap(s_base.Base):
|
|
|
1436
1441
|
if etyp == s_layer.EDIT_EDGE_ADD:
|
|
1437
1442
|
verb, n2iden = parms
|
|
1438
1443
|
n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
|
|
1439
|
-
callbacks.append((self.view.runEdgeAdd, (node, verb, n2)))
|
|
1444
|
+
callbacks.append((self.view.runEdgeAdd, (node, verb, n2, useriden)))
|
|
1440
1445
|
|
|
1441
1446
|
if etyp == s_layer.EDIT_EDGE_DEL:
|
|
1442
1447
|
verb, n2iden = parms
|
|
1443
1448
|
n2 = await self.getNodeByBuid(s_common.uhex(n2iden))
|
|
1444
|
-
callbacks.append((self.view.runEdgeDel, (node, verb, n2)))
|
|
1449
|
+
callbacks.append((self.view.runEdgeDel, (node, verb, n2, useriden)))
|
|
1445
1450
|
|
|
1446
|
-
|
|
1451
|
+
for func, args in callbacks:
|
|
1452
|
+
await func(*args)
|
|
1447
1453
|
|
|
1448
1454
|
if actualedits:
|
|
1449
1455
|
await self.fire('node:edits', edits=actualedits)
|