synapse 2.155.0__py311-none-any.whl → 2.156.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cmds/cortex.py +2 -14
- synapse/common.py +1 -28
- synapse/cortex.py +10 -510
- synapse/lib/ast.py +60 -1
- synapse/lib/cell.py +33 -8
- synapse/lib/certdir.py +11 -0
- synapse/lib/cmdr.py +0 -5
- synapse/lib/gis.py +2 -2
- synapse/lib/httpapi.py +1 -43
- synapse/lib/layer.py +64 -201
- synapse/lib/lmdbslab.py +11 -0
- synapse/lib/node.py +1 -3
- synapse/lib/parser.py +10 -0
- synapse/lib/snap.py +121 -21
- synapse/lib/storm.lark +23 -6
- synapse/lib/storm.py +15 -338
- synapse/lib/storm_format.py +5 -0
- synapse/lib/stormlib/gen.py +1 -2
- synapse/lib/stormlib/gis.py +41 -0
- synapse/lib/stormlib/stats.py +21 -2
- synapse/lib/stormlib/storm.py +16 -1
- synapse/lib/stormtypes.py +225 -12
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +96 -21
- synapse/models/inet.py +60 -30
- synapse/models/infotech.py +56 -1
- synapse/models/orgs.py +3 -0
- synapse/models/risk.py +15 -0
- synapse/models/syn.py +0 -38
- synapse/tests/test_cmds_cortex.py +1 -1
- synapse/tests/test_cortex.py +32 -336
- synapse/tests/test_lib_agenda.py +19 -54
- synapse/tests/test_lib_aha.py +97 -0
- synapse/tests/test_lib_ast.py +402 -0
- synapse/tests/test_lib_grammar.py +30 -10
- synapse/tests/test_lib_httpapi.py +0 -46
- synapse/tests/test_lib_layer.py +19 -234
- synapse/tests/test_lib_lmdbslab.py +22 -0
- synapse/tests/test_lib_snap.py +9 -0
- synapse/tests/test_lib_storm.py +16 -309
- synapse/tests/test_lib_stormlib_gis.py +21 -0
- synapse/tests/test_lib_stormlib_stats.py +107 -20
- synapse/tests/test_lib_stormlib_storm.py +25 -0
- synapse/tests/test_lib_stormtypes.py +231 -8
- synapse/tests/test_lib_view.py +6 -13
- synapse/tests/test_model_base.py +1 -1
- synapse/tests/test_model_inet.py +15 -0
- synapse/tests/test_model_infotech.py +60 -0
- synapse/tests/test_model_orgs.py +10 -0
- synapse/tests/test_model_person.py +0 -3
- synapse/tests/test_model_risk.py +20 -0
- synapse/tests/test_model_syn.py +20 -34
- synapse/tests/test_tools_csvtool.py +2 -1
- synapse/tests/test_tools_feed.py +4 -30
- synapse/tools/csvtool.py +2 -1
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/METADATA +3 -3
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/RECORD +60 -62
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/WHEEL +1 -1
- synapse/cmds/cron.py +0 -726
- synapse/cmds/trigger.py +0 -319
- synapse/tests/test_cmds_cron.py +0 -453
- synapse/tests/test_cmds_trigger.py +0 -176
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/LICENSE +0 -0
- {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py
CHANGED
|
@@ -55,6 +55,7 @@ import synapse.lib.provenance as s_provenance
|
|
|
55
55
|
import synapse.lib.stormtypes as s_stormtypes
|
|
56
56
|
|
|
57
57
|
import synapse.lib.stormlib.gen as s_stormlib_gen # NOQA
|
|
58
|
+
import synapse.lib.stormlib.gis as s_stormlib_gis # NOQA
|
|
58
59
|
import synapse.lib.stormlib.hex as s_stormlib_hex # NOQA
|
|
59
60
|
import synapse.lib.stormlib.log as s_stormlib_log # NOQA
|
|
60
61
|
import synapse.lib.stormlib.xml as s_stormlib_xml # NOQA
|
|
@@ -219,11 +220,6 @@ class CoreApi(s_cell.CellApi):
|
|
|
219
220
|
def getCoreMods(self):
|
|
220
221
|
return self.cell.getCoreMods()
|
|
221
222
|
|
|
222
|
-
def stat(self):
|
|
223
|
-
self.user.confirm(('status',))
|
|
224
|
-
s_common.deprdate('Cortex.stat() telepath API', s_common._splicedepr)
|
|
225
|
-
return self.cell.stat()
|
|
226
|
-
|
|
227
223
|
async def getModelDict(self):
|
|
228
224
|
'''
|
|
229
225
|
Return a dictionary which describes the data model.
|
|
@@ -288,228 +284,11 @@ class CoreApi(s_cell.CellApi):
|
|
|
288
284
|
opts = self._reqValidStormOpts(opts)
|
|
289
285
|
return await self.cell.feedFromAxon(sha256, opts=opts)
|
|
290
286
|
|
|
291
|
-
async def addCronJob(self, cdef):
|
|
292
|
-
'''
|
|
293
|
-
This API is deprecated.
|
|
294
|
-
|
|
295
|
-
Add a cron job to the cortex
|
|
296
|
-
|
|
297
|
-
A cron job is a persistently-stored item that causes storm queries to be run in the future. The specification
|
|
298
|
-
for the times that the queries run can be one-shot or recurring.
|
|
299
|
-
|
|
300
|
-
Args:
|
|
301
|
-
query (str): The storm query to execute in the future
|
|
302
|
-
reqs (Union[Dict[str, Union[int, List[int]]], List[Dict[...]]]):
|
|
303
|
-
Either a dict of the fixed time fields or a list of such dicts. The keys are in the set ('year',
|
|
304
|
-
'month', 'dayofmonth', 'dayofweek', 'hour', 'minute'. The values must be positive integers, except for
|
|
305
|
-
the key of 'dayofmonth' in which it may also be a negative integer which represents the number of days
|
|
306
|
-
from the end of the month with -1 representing the last day of the month. All values may also be lists
|
|
307
|
-
of valid values.
|
|
308
|
-
incunit (Optional[str]):
|
|
309
|
-
A member of the same set as above, with an additional member 'day'. If is None (default), then the
|
|
310
|
-
appointment is one-shot and will not recur.
|
|
311
|
-
incvals (Union[int, List[int]):
|
|
312
|
-
A integer or a list of integers of the number of units
|
|
313
|
-
|
|
314
|
-
Returns (bytes):
|
|
315
|
-
An iden that can be used to later modify, query, and delete the job.
|
|
316
|
-
|
|
317
|
-
Notes:
|
|
318
|
-
reqs must have fields present or incunit must not be None (or both)
|
|
319
|
-
The incunit if not None it must be larger in unit size than all the keys in all reqs elements.
|
|
320
|
-
'''
|
|
321
|
-
cdef['creator'] = self.user.iden
|
|
322
|
-
|
|
323
|
-
s_common.deprdate('Cortex.addCronJob() telepath API', s_common._splicedepr)
|
|
324
|
-
self.user.confirm(('cron', 'add'), gateiden='cortex')
|
|
325
|
-
return await self.cell.addCronJob(cdef)
|
|
326
|
-
|
|
327
|
-
async def delCronJob(self, iden):
|
|
328
|
-
'''
|
|
329
|
-
This API is deprecated.
|
|
330
|
-
|
|
331
|
-
Delete a cron job
|
|
332
|
-
|
|
333
|
-
Args:
|
|
334
|
-
iden (bytes): The iden of the cron job to be deleted
|
|
335
|
-
'''
|
|
336
|
-
s_common.deprdate('Cortex.delCronJob() telepath API', s_common._splicedepr)
|
|
337
|
-
self.user.confirm(('cron', 'del'), gateiden=iden)
|
|
338
|
-
await self.cell.delCronJob(iden)
|
|
339
|
-
|
|
340
|
-
async def updateCronJob(self, iden, query):
|
|
341
|
-
'''
|
|
342
|
-
This API is deprecated.
|
|
343
|
-
|
|
344
|
-
Change an existing cron job's query
|
|
345
|
-
|
|
346
|
-
Args:
|
|
347
|
-
iden (bytes): The iden of the cron job to be changed
|
|
348
|
-
'''
|
|
349
|
-
s_common.deprdate('Cortex.updateCronJob() telepath API', s_common._splicedepr)
|
|
350
|
-
self.user.confirm(('cron', 'set'), gateiden=iden)
|
|
351
|
-
await self.cell.updateCronJob(iden, query)
|
|
352
|
-
|
|
353
|
-
async def enableCronJob(self, iden):
|
|
354
|
-
'''
|
|
355
|
-
This API is deprecated.
|
|
356
|
-
|
|
357
|
-
Enable a cron job
|
|
358
|
-
|
|
359
|
-
Args:
|
|
360
|
-
iden (bytes): The iden of the cron job to be changed
|
|
361
|
-
'''
|
|
362
|
-
s_common.deprdate('Cortex.enableCronJob() telepath API', s_common._splicedepr)
|
|
363
|
-
self.user.confirm(('cron', 'set'), gateiden=iden)
|
|
364
|
-
await self.cell.enableCronJob(iden)
|
|
365
|
-
|
|
366
|
-
async def disableCronJob(self, iden):
|
|
367
|
-
'''
|
|
368
|
-
This API is deprecated.
|
|
369
|
-
|
|
370
|
-
Enable a cron job
|
|
371
|
-
|
|
372
|
-
Args:
|
|
373
|
-
iden (bytes): The iden of the cron job to be changed
|
|
374
|
-
'''
|
|
375
|
-
s_common.deprdate('Cortex.disableCronJob() telepath API', s_common._splicedepr)
|
|
376
|
-
self.user.confirm(('cron', 'set'), gateiden=iden)
|
|
377
|
-
await self.cell.disableCronJob(iden)
|
|
378
|
-
|
|
379
|
-
async def listCronJobs(self):
|
|
380
|
-
'''
|
|
381
|
-
This API is deprecated.
|
|
382
|
-
|
|
383
|
-
Get information about all the cron jobs accessible to the current user
|
|
384
|
-
'''
|
|
385
|
-
s_common.deprdate('Cortex.listCronJobs() telepath API', s_common._splicedepr)
|
|
386
|
-
|
|
387
|
-
crons = []
|
|
388
|
-
for cron in await self.cell.listCronJobs():
|
|
389
|
-
|
|
390
|
-
if not self.user.allowed(('cron', 'get'), gateiden=cron.get('iden')):
|
|
391
|
-
continue
|
|
392
|
-
|
|
393
|
-
crons.append(cron)
|
|
394
|
-
|
|
395
|
-
return crons
|
|
396
|
-
|
|
397
|
-
async def editCronJob(self, iden, name, valu):
|
|
398
|
-
'''
|
|
399
|
-
Update a value in a cron definition.
|
|
400
|
-
'''
|
|
401
|
-
iden = str(iden)
|
|
402
|
-
name = str(name)
|
|
403
|
-
|
|
404
|
-
s_common.deprdate('Cortex.editCronJob() telepath API', s_common._splicedepr)
|
|
405
|
-
|
|
406
|
-
if name == 'creator':
|
|
407
|
-
# this permission must be granted cortex wide
|
|
408
|
-
# to prevent abuse...
|
|
409
|
-
self.user.confirm(('cron', 'set', 'creator'))
|
|
410
|
-
|
|
411
|
-
else:
|
|
412
|
-
self.user.confirm(('cron', 'set', name), gateiden=iden)
|
|
413
|
-
|
|
414
|
-
return await self.cell.editCronJob(iden, name, valu)
|
|
415
|
-
|
|
416
|
-
async def setStormCmd(self, cdef):
|
|
417
|
-
'''
|
|
418
|
-
Set the definition of a pure storm command in the cortex.
|
|
419
|
-
'''
|
|
420
|
-
s_common.deprdate('Cortex.setStormCmd() telepath API', s_common._splicedepr)
|
|
421
|
-
self.user.confirm(('admin', 'cmds'))
|
|
422
|
-
return await self.cell.setStormCmd(cdef)
|
|
423
|
-
|
|
424
|
-
async def delStormCmd(self, name):
|
|
425
|
-
'''
|
|
426
|
-
Remove a pure storm command from the cortex.
|
|
427
|
-
'''
|
|
428
|
-
s_common.deprdate('Cortex.delStormCmd() telepath API', s_common._splicedepr)
|
|
429
|
-
self.user.confirm(('admin', 'cmds'))
|
|
430
|
-
return await self.cell.delStormCmd(name)
|
|
431
|
-
|
|
432
287
|
async def _reqDefLayerAllowed(self, perms):
|
|
433
288
|
view = self.cell.getView()
|
|
434
289
|
wlyr = view.layers[0]
|
|
435
290
|
self.user.confirm(perms, gateiden=wlyr.iden)
|
|
436
291
|
|
|
437
|
-
async def addNodeTag(self, iden, tag, valu=(None, None)):
|
|
438
|
-
'''
|
|
439
|
-
This API is deprecated.
|
|
440
|
-
|
|
441
|
-
Add a tag to a node specified by iden.
|
|
442
|
-
|
|
443
|
-
Args:
|
|
444
|
-
iden (str): A hex encoded node BUID.
|
|
445
|
-
tag (str): A tag string.
|
|
446
|
-
valu (tuple): A time interval tuple or (None, None).
|
|
447
|
-
'''
|
|
448
|
-
s_common.deprdate('Cortex.addNodeTag() telepath API', s_common._splicedepr)
|
|
449
|
-
await self._reqDefLayerAllowed(('node', 'tag', 'add', *tag.split('.')))
|
|
450
|
-
return await self.cell.addNodeTag(self.user, iden, tag, valu)
|
|
451
|
-
|
|
452
|
-
async def delNodeTag(self, iden, tag):
|
|
453
|
-
'''
|
|
454
|
-
This API is deprecated.
|
|
455
|
-
|
|
456
|
-
Delete a tag from the node specified by iden. Deprecated in 2.0.0.
|
|
457
|
-
|
|
458
|
-
Args:
|
|
459
|
-
iden (str): A hex encoded node BUID.
|
|
460
|
-
tag (str): A tag string.
|
|
461
|
-
'''
|
|
462
|
-
s_common.deprdate('Cortex.delNodeTag() telepath API', s_common._splicedepr)
|
|
463
|
-
await self._reqDefLayerAllowed(('node', 'tag', 'del', *tag.split('.')))
|
|
464
|
-
return await self.cell.delNodeTag(self.user, iden, tag)
|
|
465
|
-
|
|
466
|
-
async def setNodeProp(self, iden, name, valu):
|
|
467
|
-
'''
|
|
468
|
-
This API is deprecated.
|
|
469
|
-
|
|
470
|
-
Set a property on a single node.
|
|
471
|
-
'''
|
|
472
|
-
s_common.deprdate('Cortex.setNodeProp() telepath API', s_common._splicedepr)
|
|
473
|
-
buid = s_common.uhex(iden)
|
|
474
|
-
|
|
475
|
-
async with await self.cell.snap(user=self.user) as snap:
|
|
476
|
-
|
|
477
|
-
with s_provenance.claim('coreapi', meth='prop:set', user=snap.user.iden):
|
|
478
|
-
|
|
479
|
-
node = await snap.getNodeByBuid(buid)
|
|
480
|
-
if node is None:
|
|
481
|
-
raise s_exc.NoSuchIden(iden=iden)
|
|
482
|
-
|
|
483
|
-
prop = node.form.props.get(name)
|
|
484
|
-
self.user.confirm(('node', 'prop', 'set', prop.full), gateiden=snap.wlyr.iden)
|
|
485
|
-
|
|
486
|
-
await node.set(name, valu)
|
|
487
|
-
return node.pack()
|
|
488
|
-
|
|
489
|
-
async def delNodeProp(self, iden, name):
|
|
490
|
-
'''
|
|
491
|
-
This API is deprecated.
|
|
492
|
-
|
|
493
|
-
Delete a property from a single node.
|
|
494
|
-
'''
|
|
495
|
-
s_common.deprdate('Cortex.delNodeProp() telepath API', s_common._splicedepr)
|
|
496
|
-
buid = s_common.uhex(iden)
|
|
497
|
-
|
|
498
|
-
async with await self.cell.snap(user=self.user) as snap:
|
|
499
|
-
|
|
500
|
-
with s_provenance.claim('coreapi', meth='prop:del', user=snap.user.iden):
|
|
501
|
-
|
|
502
|
-
node = await snap.getNodeByBuid(buid)
|
|
503
|
-
if node is None:
|
|
504
|
-
raise s_exc.NoSuchIden(iden=iden)
|
|
505
|
-
|
|
506
|
-
prop = node.form.props.get(name)
|
|
507
|
-
|
|
508
|
-
self.user.confirm(('node', 'prop', 'del', prop.full), gateiden=snap.wlyr.iden)
|
|
509
|
-
|
|
510
|
-
await node.pop(name)
|
|
511
|
-
return node.pack()
|
|
512
|
-
|
|
513
292
|
async def addNode(self, form, valu, props=None):
|
|
514
293
|
'''
|
|
515
294
|
Deprecated in 2.0.0.
|
|
@@ -610,18 +389,6 @@ class CoreApi(s_cell.CellApi):
|
|
|
610
389
|
opts = self._reqValidStormOpts(opts)
|
|
611
390
|
return await self.cell.count(text, opts=opts)
|
|
612
391
|
|
|
613
|
-
async def eval(self, text, opts=None):
|
|
614
|
-
'''
|
|
615
|
-
This API is deprecated.
|
|
616
|
-
|
|
617
|
-
Evaluate a storm query and yield packed nodes.
|
|
618
|
-
'''
|
|
619
|
-
s_common.deprdate('Cortex.eval() telepath API', s_common._splicedepr)
|
|
620
|
-
opts = self._reqValidStormOpts(opts)
|
|
621
|
-
view = self.cell._viewFromOpts(opts)
|
|
622
|
-
async for pode in view.iterStormPodes(text, opts=opts):
|
|
623
|
-
yield pode
|
|
624
|
-
|
|
625
392
|
async def storm(self, text, opts=None):
|
|
626
393
|
'''
|
|
627
394
|
Evaluate a storm query and yield result messages.
|
|
@@ -650,26 +417,6 @@ class CoreApi(s_cell.CellApi):
|
|
|
650
417
|
'''
|
|
651
418
|
return await self.cell.reqValidStorm(text, opts)
|
|
652
419
|
|
|
653
|
-
async def watch(self, wdef):
|
|
654
|
-
'''
|
|
655
|
-
This API is deprecated.
|
|
656
|
-
|
|
657
|
-
Hook cortex/view/layer watch points based on a specified watch definition.
|
|
658
|
-
|
|
659
|
-
Example:
|
|
660
|
-
|
|
661
|
-
wdef = { 'tags': [ 'foo.bar', 'baz.*' ] }
|
|
662
|
-
|
|
663
|
-
async for mesg in core.watch(wdef):
|
|
664
|
-
dostuff(mesg)
|
|
665
|
-
'''
|
|
666
|
-
s_common.deprdate('Cortex.watch() telepath API', s_common._splicedepr)
|
|
667
|
-
iden = wdef.get('view', self.cell.view.iden)
|
|
668
|
-
self.user.confirm(('watch',), gateiden=iden)
|
|
669
|
-
|
|
670
|
-
async for mesg in self.cell.watch(wdef):
|
|
671
|
-
yield mesg
|
|
672
|
-
|
|
673
420
|
async def syncLayerNodeEdits(self, offs, layriden=None, wait=True):
|
|
674
421
|
'''
|
|
675
422
|
Yield (indx, mesg) nodeedit sets for the given layer beginning at offset.
|
|
@@ -687,49 +434,6 @@ class CoreApi(s_cell.CellApi):
|
|
|
687
434
|
async for item in self.cell.syncLayerNodeEdits(layr.iden, offs, wait=wait):
|
|
688
435
|
yield item
|
|
689
436
|
|
|
690
|
-
@s_cell.adminapi()
|
|
691
|
-
async def splices(self, offs=None, size=None, layriden=None):
|
|
692
|
-
'''
|
|
693
|
-
This API is deprecated.
|
|
694
|
-
|
|
695
|
-
Return the list of splices at the given offset.
|
|
696
|
-
'''
|
|
697
|
-
s_common.deprdate('Cortex.splices() telepath API', s_common._splicedepr)
|
|
698
|
-
layr = self.cell.getLayer(layriden)
|
|
699
|
-
count = 0
|
|
700
|
-
async for mesg in layr.splices(offs=offs, size=size):
|
|
701
|
-
count += 1
|
|
702
|
-
if not count % 1000:
|
|
703
|
-
await asyncio.sleep(0)
|
|
704
|
-
yield mesg
|
|
705
|
-
|
|
706
|
-
@s_cell.adminapi()
|
|
707
|
-
async def splicesBack(self, offs=None, size=None):
|
|
708
|
-
'''
|
|
709
|
-
This API is deprecated.
|
|
710
|
-
|
|
711
|
-
Return the list of splices backwards from the given offset.
|
|
712
|
-
'''
|
|
713
|
-
s_common.deprdate('Cortex.splicesBack() telepath API', s_common._splicedepr)
|
|
714
|
-
count = 0
|
|
715
|
-
async for mesg in self.cell.view.layers[0].splicesBack(offs=offs, size=size):
|
|
716
|
-
count += 1
|
|
717
|
-
if not count % 1000: # pragma: no cover
|
|
718
|
-
await asyncio.sleep(0)
|
|
719
|
-
yield mesg
|
|
720
|
-
|
|
721
|
-
async def spliceHistory(self):
|
|
722
|
-
'''
|
|
723
|
-
This API is deprecated.
|
|
724
|
-
|
|
725
|
-
Yield splices backwards from the end of the splice log.
|
|
726
|
-
|
|
727
|
-
Will only return the user's own splices unless they are an admin.
|
|
728
|
-
'''
|
|
729
|
-
s_common.deprdate('Cortex.spliceHistory() telepath API', s_common._splicedepr)
|
|
730
|
-
async for splice in self.cell.spliceHistory(self.user):
|
|
731
|
-
yield splice
|
|
732
|
-
|
|
733
437
|
async def getPropNorm(self, prop, valu):
|
|
734
438
|
'''
|
|
735
439
|
Get the normalized property value based on the Cortex data model.
|
|
@@ -1184,7 +888,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1184
888
|
self.viewsbylayer = collections.defaultdict(list)
|
|
1185
889
|
|
|
1186
890
|
self.modules = {}
|
|
1187
|
-
self.splicers = {}
|
|
1188
891
|
self.feedfuncs = {}
|
|
1189
892
|
self.stormcmds = {}
|
|
1190
893
|
|
|
@@ -1238,7 +941,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1238
941
|
self.onfini(self._onCoreFini)
|
|
1239
942
|
|
|
1240
943
|
await self._initCoreHive()
|
|
1241
|
-
self._initSplicers()
|
|
1242
944
|
self._initStormLibs()
|
|
1243
945
|
self._initFeedFuncs()
|
|
1244
946
|
|
|
@@ -1641,6 +1343,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1641
1343
|
'desc': 'Controls access to add a new view including forks.'},
|
|
1642
1344
|
{'perm': ('view', 'del'), 'gate': 'view',
|
|
1643
1345
|
'desc': 'Controls access to delete a view.'},
|
|
1346
|
+
{'perm': ('view', 'fork'), 'gate': 'view', 'default': True,
|
|
1347
|
+
'desc': 'Controls access to fork a view.'},
|
|
1644
1348
|
{'perm': ('view', 'read'), 'gate': 'view',
|
|
1645
1349
|
'desc': 'Controls read access to view.'},
|
|
1646
1350
|
{'perm': ('view', 'set', '<setting>'), 'gate': 'view',
|
|
@@ -3296,45 +3000,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3296
3000
|
except Exception as e:
|
|
3297
3001
|
logger.warning(f'ext tag prop ({prop}) error: {e}')
|
|
3298
3002
|
|
|
3299
|
-
@contextlib.asynccontextmanager
|
|
3300
|
-
async def watcher(self, wdef):
|
|
3301
|
-
|
|
3302
|
-
iden = wdef.get('view', self.view.iden)
|
|
3303
|
-
|
|
3304
|
-
view = self.views.get(iden)
|
|
3305
|
-
if view is None:
|
|
3306
|
-
raise s_exc.NoSuchView(mesg=f'No such view {iden=}', iden=iden)
|
|
3307
|
-
|
|
3308
|
-
async with await s_queue.Window.anit(maxsize=10000) as wind:
|
|
3309
|
-
|
|
3310
|
-
tags = wdef.get('tags')
|
|
3311
|
-
if tags is not None:
|
|
3312
|
-
|
|
3313
|
-
tglobs = s_cache.TagGlobs()
|
|
3314
|
-
[tglobs.add(t, True) for t in tags]
|
|
3315
|
-
|
|
3316
|
-
async def ontag(mesg):
|
|
3317
|
-
name = mesg[1].get('tag')
|
|
3318
|
-
if not tglobs.get(name):
|
|
3319
|
-
return
|
|
3320
|
-
|
|
3321
|
-
await wind.put(mesg)
|
|
3322
|
-
|
|
3323
|
-
for layr in self.view.layers:
|
|
3324
|
-
layr.on('tag:add', ontag, base=wind)
|
|
3325
|
-
layr.on('tag:del', ontag, base=wind)
|
|
3326
|
-
|
|
3327
|
-
yield wind
|
|
3328
|
-
|
|
3329
|
-
async def watch(self, wdef):
|
|
3330
|
-
'''
|
|
3331
|
-
Hook cortex/view/layer watch points based on a specified watch definition.
|
|
3332
|
-
( see CoreApi.watch() docs for details )
|
|
3333
|
-
'''
|
|
3334
|
-
async with self.watcher(wdef) as wind:
|
|
3335
|
-
async for mesg in wind:
|
|
3336
|
-
yield mesg
|
|
3337
|
-
|
|
3338
3003
|
async def getExtModel(self):
|
|
3339
3004
|
'''
|
|
3340
3005
|
Get all extended model properties in the Cortex.
|
|
@@ -3964,23 +3629,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
3964
3629
|
|
|
3965
3630
|
await self.waitfini(1)
|
|
3966
3631
|
|
|
3967
|
-
async def spliceHistory(self, user):
|
|
3968
|
-
'''
|
|
3969
|
-
Yield splices backwards from the end of the nodeedit log.
|
|
3970
|
-
|
|
3971
|
-
Will only return user's own splices unless they are an admin.
|
|
3972
|
-
'''
|
|
3973
|
-
layr = self.view.layers[0]
|
|
3974
|
-
|
|
3975
|
-
count = 0
|
|
3976
|
-
async for _, mesg in layr.splicesBack():
|
|
3977
|
-
count += 1
|
|
3978
|
-
if not count % 1000: # pragma: no cover
|
|
3979
|
-
await asyncio.sleep(0)
|
|
3980
|
-
|
|
3981
|
-
if user.iden == mesg[1]['user'] or user.isAdmin():
|
|
3982
|
-
yield mesg
|
|
3983
|
-
|
|
3984
3632
|
async def _initCoreHive(self):
|
|
3985
3633
|
stormvarsnode = await self.hive.open(('cortex', 'storm', 'vars'))
|
|
3986
3634
|
self.stormvars = await stormvarsnode.dict()
|
|
@@ -4158,7 +3806,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4158
3806
|
self.addStormCmd(s_storm.HelpCmd)
|
|
4159
3807
|
self.addStormCmd(s_storm.IdenCmd)
|
|
4160
3808
|
self.addStormCmd(s_storm.SpinCmd)
|
|
4161
|
-
self.addStormCmd(s_storm.SudoCmd)
|
|
4162
3809
|
self.addStormCmd(s_storm.UniqCmd)
|
|
4163
3810
|
self.addStormCmd(s_storm.BatchCmd)
|
|
4164
3811
|
self.addStormCmd(s_storm.CountCmd)
|
|
@@ -4181,8 +3828,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4181
3828
|
self.addStormCmd(s_storm.IntersectCmd)
|
|
4182
3829
|
self.addStormCmd(s_storm.MoveNodesCmd)
|
|
4183
3830
|
self.addStormCmd(s_storm.BackgroundCmd)
|
|
4184
|
-
self.addStormCmd(s_storm.SpliceListCmd)
|
|
4185
|
-
self.addStormCmd(s_storm.SpliceUndoCmd)
|
|
4186
3831
|
self.addStormCmd(s_stormlib_macro.MacroExecCmd)
|
|
4187
3832
|
self.addStormCmd(s_stormlib_stats.StatsCountByCmd)
|
|
4188
3833
|
|
|
@@ -4242,29 +3887,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4242
3887
|
if path:
|
|
4243
3888
|
self.addStormLib(path, ctor)
|
|
4244
3889
|
|
|
4245
|
-
def _initSplicers(self):
|
|
4246
|
-
'''
|
|
4247
|
-
Registration for splice handlers.
|
|
4248
|
-
'''
|
|
4249
|
-
splicers = {
|
|
4250
|
-
'tag:add': self._onFeedTagAdd,
|
|
4251
|
-
'tag:del': self._onFeedTagDel,
|
|
4252
|
-
'node:add': self._onFeedNodeAdd,
|
|
4253
|
-
'node:del': self._onFeedNodeDel,
|
|
4254
|
-
'prop:set': self._onFeedPropSet,
|
|
4255
|
-
'prop:del': self._onFeedPropDel,
|
|
4256
|
-
'tag:prop:set': self._onFeedTagPropSet,
|
|
4257
|
-
'tag:prop:del': self._onFeedTagPropDel,
|
|
4258
|
-
}
|
|
4259
|
-
self.splicers.update(**splicers)
|
|
4260
|
-
|
|
4261
3890
|
def _initFeedFuncs(self):
|
|
4262
3891
|
'''
|
|
4263
3892
|
Registration for built-in Cortex feed functions.
|
|
4264
3893
|
'''
|
|
4265
3894
|
self.setFeedFunc('syn.nodes', self._addSynNodes)
|
|
4266
|
-
self.setFeedFunc('syn.splice', self._addSynSplice)
|
|
4267
|
-
self.setFeedFunc('syn.nodeedits', self._addSynNodeEdits)
|
|
4268
3895
|
|
|
4269
3896
|
def _initCortexHttpApi(self):
|
|
4270
3897
|
'''
|
|
@@ -4272,7 +3899,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4272
3899
|
'''
|
|
4273
3900
|
self.addHttpApi('/api/v1/feed', s_httpapi.FeedV1, {'cell': self})
|
|
4274
3901
|
self.addHttpApi('/api/v1/storm', s_httpapi.StormV1, {'cell': self})
|
|
4275
|
-
self.addHttpApi('/api/v1/watch', s_httpapi.WatchSockV1, {'cell': self})
|
|
4276
3902
|
self.addHttpApi('/api/v1/storm/call', s_httpapi.StormCallV1, {'cell': self})
|
|
4277
3903
|
self.addHttpApi('/api/v1/storm/nodes', s_httpapi.StormNodesV1, {'cell': self})
|
|
4278
3904
|
self.addHttpApi('/api/v1/storm/export', s_httpapi.StormExportV1, {'cell': self})
|
|
@@ -4694,6 +4320,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4694
4320
|
async def addView(self, vdef, nexs=True):
|
|
4695
4321
|
|
|
4696
4322
|
vdef['iden'] = s_common.guid()
|
|
4323
|
+
vdef['created'] = s_common.now()
|
|
4324
|
+
|
|
4697
4325
|
vdef.setdefault('parent', None)
|
|
4698
4326
|
vdef.setdefault('worldreadable', False)
|
|
4699
4327
|
vdef.setdefault('creator', self.auth.rootuser.iden)
|
|
@@ -4888,10 +4516,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4888
4516
|
|
|
4889
4517
|
return view
|
|
4890
4518
|
|
|
4891
|
-
def reqView(self, iden):
|
|
4519
|
+
def reqView(self, iden, mesg=None):
|
|
4892
4520
|
view = self.getView(iden)
|
|
4893
4521
|
if view is None: # pragma: no cover
|
|
4894
|
-
mesg
|
|
4522
|
+
if mesg is None:
|
|
4523
|
+
mesg = f'No view with iden: {iden}'
|
|
4895
4524
|
raise s_exc.NoSuchView(mesg=mesg, iden=iden)
|
|
4896
4525
|
return view
|
|
4897
4526
|
|
|
@@ -4931,6 +4560,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4931
4560
|
ldef = ldef or {}
|
|
4932
4561
|
|
|
4933
4562
|
ldef['iden'] = s_common.guid()
|
|
4563
|
+
ldef['created'] = s_common.now()
|
|
4564
|
+
|
|
4934
4565
|
ldef.setdefault('creator', self.auth.rootuser.iden)
|
|
4935
4566
|
ldef.setdefault('lockmemory', self.conf.get('layers:lockmemory'))
|
|
4936
4567
|
ldef.setdefault('logedits', self.conf.get('layers:logedits'))
|
|
@@ -5476,117 +5107,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5476
5107
|
async for node in snap.addNodes(items):
|
|
5477
5108
|
pass
|
|
5478
5109
|
|
|
5479
|
-
async def _addSynSplice(self, snap, items):
|
|
5480
|
-
s_common.deprdate('Cortex.addFeedData(syn.splice, ...) API', s_common._splicedepr)
|
|
5481
|
-
|
|
5482
|
-
for item in items:
|
|
5483
|
-
func = self.splicers.get(item[0])
|
|
5484
|
-
|
|
5485
|
-
if func is None:
|
|
5486
|
-
await snap.warn(f'no such splice: {item!r}')
|
|
5487
|
-
continue
|
|
5488
|
-
|
|
5489
|
-
try:
|
|
5490
|
-
await func(snap, item)
|
|
5491
|
-
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
|
|
5492
|
-
raise
|
|
5493
|
-
except Exception as e:
|
|
5494
|
-
logger.exception('splice error')
|
|
5495
|
-
await snap.warn(f'splice error: {e}')
|
|
5496
|
-
|
|
5497
|
-
async def _onFeedNodeAdd(self, snap, mesg):
|
|
5498
|
-
|
|
5499
|
-
ndef = mesg[1].get('ndef')
|
|
5500
|
-
|
|
5501
|
-
if ndef is None:
|
|
5502
|
-
await snap.warn(f'Invalid Splice: {mesg!r}')
|
|
5503
|
-
return
|
|
5504
|
-
|
|
5505
|
-
await snap.addNode(*ndef)
|
|
5506
|
-
|
|
5507
|
-
async def _onFeedNodeDel(self, snap, mesg):
|
|
5508
|
-
|
|
5509
|
-
ndef = mesg[1].get('ndef')
|
|
5510
|
-
|
|
5511
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5512
|
-
if node is None:
|
|
5513
|
-
return
|
|
5514
|
-
|
|
5515
|
-
await node.delete()
|
|
5516
|
-
|
|
5517
|
-
async def _onFeedPropSet(self, snap, mesg):
|
|
5518
|
-
|
|
5519
|
-
ndef = mesg[1].get('ndef')
|
|
5520
|
-
name = mesg[1].get('prop')
|
|
5521
|
-
valu = mesg[1].get('valu')
|
|
5522
|
-
|
|
5523
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5524
|
-
if node is None:
|
|
5525
|
-
return
|
|
5526
|
-
|
|
5527
|
-
await node.set(name, valu)
|
|
5528
|
-
|
|
5529
|
-
async def _onFeedPropDel(self, snap, mesg):
|
|
5530
|
-
|
|
5531
|
-
ndef = mesg[1].get('ndef')
|
|
5532
|
-
name = mesg[1].get('prop')
|
|
5533
|
-
|
|
5534
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5535
|
-
if node is None:
|
|
5536
|
-
return
|
|
5537
|
-
|
|
5538
|
-
await node.pop(name)
|
|
5539
|
-
|
|
5540
|
-
async def _onFeedTagAdd(self, snap, mesg):
|
|
5541
|
-
|
|
5542
|
-
ndef = mesg[1].get('ndef')
|
|
5543
|
-
tag = mesg[1].get('tag')
|
|
5544
|
-
valu = mesg[1].get('valu')
|
|
5545
|
-
|
|
5546
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5547
|
-
if node is None:
|
|
5548
|
-
return
|
|
5549
|
-
|
|
5550
|
-
await node.addTag(tag, valu=valu)
|
|
5551
|
-
|
|
5552
|
-
async def _onFeedTagDel(self, snap, mesg):
|
|
5553
|
-
|
|
5554
|
-
ndef = mesg[1].get('ndef')
|
|
5555
|
-
tag = mesg[1].get('tag')
|
|
5556
|
-
|
|
5557
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5558
|
-
if node is None:
|
|
5559
|
-
return
|
|
5560
|
-
|
|
5561
|
-
await node.delTag(tag)
|
|
5562
|
-
|
|
5563
|
-
async def _onFeedTagPropSet(self, snap, mesg):
|
|
5564
|
-
|
|
5565
|
-
tag = mesg[1].get('tag')
|
|
5566
|
-
prop = mesg[1].get('prop')
|
|
5567
|
-
ndef = mesg[1].get('ndef')
|
|
5568
|
-
valu = mesg[1].get('valu')
|
|
5569
|
-
|
|
5570
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5571
|
-
if node is not None:
|
|
5572
|
-
await node.setTagProp(tag, prop, valu)
|
|
5573
|
-
|
|
5574
|
-
async def _onFeedTagPropDel(self, snap, mesg):
|
|
5575
|
-
|
|
5576
|
-
tag = mesg[1].get('tag')
|
|
5577
|
-
prop = mesg[1].get('prop')
|
|
5578
|
-
ndef = mesg[1].get('ndef')
|
|
5579
|
-
|
|
5580
|
-
node = await snap.getNodeByNdef(ndef)
|
|
5581
|
-
if node is not None:
|
|
5582
|
-
await node.delTagProp(tag, prop)
|
|
5583
|
-
|
|
5584
|
-
async def _addSynNodeEdits(self, snap, items):
|
|
5585
|
-
s_common.deprdate('Cortex.addFeedData(syn.nodeedits, ...) API', s_common._splicedepr)
|
|
5586
|
-
for item in items:
|
|
5587
|
-
item = s_common.unjsonsafe_nodeedits(item)
|
|
5588
|
-
await snap.saveNodeEdits(item, None)
|
|
5589
|
-
|
|
5590
5110
|
async def setUserLocked(self, iden, locked):
|
|
5591
5111
|
retn = await s_cell.Cell.setUserLocked(self, iden, locked)
|
|
5592
5112
|
await self._bumpUserDmons(iden)
|
|
@@ -5777,18 +5297,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5777
5297
|
view = self._viewFromOpts(opts)
|
|
5778
5298
|
return await view.nodes(text, opts=opts)
|
|
5779
5299
|
|
|
5780
|
-
async def eval(self, text, opts=None):
|
|
5781
|
-
'''
|
|
5782
|
-
This API is deprecated.
|
|
5783
|
-
|
|
5784
|
-
Evaluate a storm query and yield packed nodes.
|
|
5785
|
-
'''
|
|
5786
|
-
s_common.deprdate('Cortex.eval() API', s_common._splicedepr)
|
|
5787
|
-
opts = self._initStormOpts(opts)
|
|
5788
|
-
view = self._viewFromOpts(opts)
|
|
5789
|
-
async for node in view.eval(text, opts=opts):
|
|
5790
|
-
yield node
|
|
5791
|
-
|
|
5792
5300
|
async def stormlist(self, text, opts=None):
|
|
5793
5301
|
return [m async for m in self.storm(text, opts=opts)]
|
|
5794
5302
|
|
|
@@ -6085,14 +5593,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
6085
5593
|
logger.exception('mod load fail: %s' % (ctor,))
|
|
6086
5594
|
return None
|
|
6087
5595
|
|
|
6088
|
-
async def stat(self):
|
|
6089
|
-
stats = {
|
|
6090
|
-
'iden': self.iden,
|
|
6091
|
-
'layer': await self.getLayer().stat(),
|
|
6092
|
-
'formcounts': await self.getFormCounts(),
|
|
6093
|
-
}
|
|
6094
|
-
return stats
|
|
6095
|
-
|
|
6096
5596
|
async def getPropNorm(self, prop, valu):
|
|
6097
5597
|
'''
|
|
6098
5598
|
Get the normalized property value based on the Cortex data model.
|