synapse 2.187.0__py311-none-any.whl → 2.188.1__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +131 -7
- synapse/datamodel.py +20 -4
- synapse/exc.py +14 -1
- synapse/lib/ast.py +6 -4
- synapse/lib/auth.py +9 -0
- synapse/lib/drive.py +1 -1
- synapse/lib/httpapi.py +2 -1
- synapse/lib/nexus.py +6 -0
- synapse/lib/node.py +5 -3
- synapse/lib/scrape.py +18 -104
- synapse/lib/storm.py +44 -28
- synapse/lib/stormlib/modelext.py +31 -0
- synapse/lib/stormlib/scrape.py +1 -4
- synapse/lib/stormtypes.py +17 -1
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +9 -3
- synapse/models/base.py +27 -0
- synapse/models/files.py +22 -0
- synapse/models/inet.py +49 -4
- synapse/models/orgs.py +64 -2
- synapse/models/proj.py +1 -6
- synapse/models/risk.py +65 -0
- synapse/tests/test_cortex.py +21 -0
- synapse/tests/test_lib_agenda.py +13 -0
- synapse/tests/test_lib_auth.py +15 -0
- synapse/tests/test_lib_cell.py +170 -159
- synapse/tests/test_lib_httpapi.py +6 -0
- synapse/tests/test_lib_nexus.py +26 -0
- synapse/tests/test_lib_scrape.py +14 -6
- synapse/tests/test_lib_storm.py +48 -0
- synapse/tests/test_lib_stormlib_modelext.py +76 -1
- synapse/tests/test_lib_stormlib_scrape.py +0 -8
- synapse/tests/test_lib_stormtypes.py +1 -1
- synapse/tests/test_lib_trigger.py +8 -0
- synapse/tests/test_lib_view.py +24 -0
- synapse/tests/test_model_base.py +11 -0
- synapse/tests/test_model_files.py +19 -0
- synapse/tests/test_model_inet.py +33 -0
- synapse/tests/test_model_orgs.py +39 -0
- synapse/tests/test_model_proj.py +11 -1
- synapse/tests/test_model_risk.py +32 -0
- {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/METADATA +1 -1
- {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/RECORD +46 -46
- {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/WHEEL +1 -1
- {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/LICENSE +0 -0
- {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/top_level.txt +0 -0
synapse/lib/scrape.py
CHANGED
|
@@ -24,8 +24,6 @@ ipaddress = s_common.ipaddress
|
|
|
24
24
|
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
26
26
|
|
|
27
|
-
SCRAPE_SPAWN_LENGTH = 5000
|
|
28
|
-
|
|
29
27
|
tldlist = list(s_data.get('iana.tlds'))
|
|
30
28
|
tldlist.extend([
|
|
31
29
|
'bit',
|
|
@@ -445,6 +443,9 @@ def _rewriteRawValu(text: str, offsets: dict, info: dict):
|
|
|
445
443
|
info['match'] = match
|
|
446
444
|
info['offset'] = baseoff + offset
|
|
447
445
|
|
|
446
|
+
def _genMatchList(text: str, regx: regex.Regex, opts: dict):
|
|
447
|
+
return [info for info in _genMatches(text, regx, opts)]
|
|
448
|
+
|
|
448
449
|
def _genMatches(text: str, regx: regex.Regex, opts: dict):
|
|
449
450
|
|
|
450
451
|
cb = opts.get('callback')
|
|
@@ -504,23 +505,9 @@ def genMatches(text: str, regx: regex.Regex, opts: dict):
|
|
|
504
505
|
for match in _genMatches(text, regx, opts):
|
|
505
506
|
yield match
|
|
506
507
|
|
|
507
|
-
def _spawn_genmatches(sock, text, regx, opts): # pragma: no cover
|
|
508
|
-
'''
|
|
509
|
-
Multiprocessing target for generating matches.
|
|
510
|
-
'''
|
|
511
|
-
try:
|
|
512
|
-
for info in _genMatches(text, regx, opts):
|
|
513
|
-
sock.sendall(s_msgpack.en((True, info)))
|
|
514
|
-
|
|
515
|
-
sock.sendall(s_msgpack.en((True, None)))
|
|
516
|
-
|
|
517
|
-
except Exception as e:
|
|
518
|
-
mesg = s_common.retnexc(e)
|
|
519
|
-
sock.sendall(s_msgpack.en(mesg))
|
|
520
|
-
|
|
521
508
|
async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
|
|
522
509
|
'''
|
|
523
|
-
Generate regular expression matches for a blob of text,
|
|
510
|
+
Generate regular expression matches for a blob of text, using the shared forked process pool.
|
|
524
511
|
|
|
525
512
|
Args:
|
|
526
513
|
text (str): The text to generate matches for.
|
|
@@ -547,30 +534,9 @@ async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
|
|
|
547
534
|
Yields:
|
|
548
535
|
dict: A dictionary of match results.
|
|
549
536
|
'''
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
return
|
|
554
|
-
|
|
555
|
-
link00, sock00 = await s_link.linksock()
|
|
556
|
-
|
|
557
|
-
try:
|
|
558
|
-
async with link00:
|
|
559
|
-
|
|
560
|
-
todo = s_common.todo(_spawn_genmatches, sock00, text, regx, opts)
|
|
561
|
-
link00.schedCoro(s_coro.spawn(todo, log_conf=s_common._getLogConfFromEnv()))
|
|
562
|
-
|
|
563
|
-
while (mesg := await link00.rx()) is not None:
|
|
564
|
-
|
|
565
|
-
info = s_common.result(mesg)
|
|
566
|
-
if info is None:
|
|
567
|
-
return
|
|
568
|
-
|
|
569
|
-
yield info
|
|
570
|
-
|
|
571
|
-
finally:
|
|
572
|
-
sock00.close()
|
|
573
|
-
|
|
537
|
+
matches = await s_coro.semafork(_genMatchList, text, regx, opts)
|
|
538
|
+
for info in matches:
|
|
539
|
+
yield info
|
|
574
540
|
|
|
575
541
|
def _contextMatches(scrape_text, text, ruletype, refang, offsets):
|
|
576
542
|
|
|
@@ -585,33 +551,16 @@ def _contextMatches(scrape_text, text, ruletype, refang, offsets):
|
|
|
585
551
|
|
|
586
552
|
yield info
|
|
587
553
|
|
|
588
|
-
def
|
|
589
|
-
|
|
590
|
-
offsets = {}
|
|
591
|
-
if refang:
|
|
592
|
-
scrape_text, offsets = refang_text2(text)
|
|
593
|
-
|
|
594
|
-
for ruletype, blobs in _regexes.items():
|
|
595
|
-
if form and form != ruletype:
|
|
596
|
-
continue
|
|
554
|
+
def _contextScrapeList(text, form=None, refang=True, first=False):
|
|
555
|
+
return [info for info in _contextScrape(text, form=form, refang=refang, first=first)]
|
|
597
556
|
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
yield info
|
|
601
|
-
|
|
602
|
-
if first:
|
|
603
|
-
return
|
|
604
|
-
|
|
605
|
-
async def _contextScrapeAsync(text, form=None, refang=True, first=False):
|
|
557
|
+
def _contextScrape(text, form=None, refang=True, first=False):
|
|
606
558
|
scrape_text = text
|
|
607
559
|
offsets = {}
|
|
608
560
|
if refang:
|
|
609
561
|
scrape_text, offsets = refang_text2(text)
|
|
610
562
|
|
|
611
563
|
for ruletype, blobs in _regexes.items():
|
|
612
|
-
|
|
613
|
-
await asyncio.sleep(0)
|
|
614
|
-
|
|
615
564
|
if form and form != ruletype:
|
|
616
565
|
continue
|
|
617
566
|
|
|
@@ -666,27 +615,12 @@ def scrape(text, ptype=None, refang=True, first=False):
|
|
|
666
615
|
Returns:
|
|
667
616
|
(str, object): Yield tuples of node ndef values.
|
|
668
617
|
'''
|
|
669
|
-
|
|
670
|
-
for info in contextScrape(text, form=ptype, refang=refang, first=first):
|
|
618
|
+
for info in _contextScrape(text, form=ptype, refang=refang, first=first):
|
|
671
619
|
yield info.get('form'), info.get('valu')
|
|
672
620
|
|
|
673
|
-
def _spawn_scrape(sock, text, form=None, refang=True, first=False): # pragma: no cover
|
|
674
|
-
'''
|
|
675
|
-
Multiprocessing target for scraping text.
|
|
676
|
-
'''
|
|
677
|
-
try:
|
|
678
|
-
for info in _contextScrape(text, form=form, refang=refang, first=first):
|
|
679
|
-
sock.sendall(s_msgpack.en((True, info)))
|
|
680
|
-
|
|
681
|
-
sock.sendall(s_msgpack.en((True, None)))
|
|
682
|
-
|
|
683
|
-
except Exception as e:
|
|
684
|
-
mesg = s_common.retnexc(e)
|
|
685
|
-
sock.sendall(s_msgpack.en(mesg))
|
|
686
|
-
|
|
687
621
|
async def contextScrapeAsync(text, form=None, refang=True, first=False):
|
|
688
622
|
'''
|
|
689
|
-
Scrape types from a blob of text and yield info dictionaries,
|
|
623
|
+
Scrape types from a blob of text and yield info dictionaries, using the shared forked process pool.
|
|
690
624
|
|
|
691
625
|
Args:
|
|
692
626
|
text (str): Text to scrape.
|
|
@@ -712,33 +646,13 @@ async def contextScrapeAsync(text, form=None, refang=True, first=False):
|
|
|
712
646
|
Returns:
|
|
713
647
|
(dict): Yield info dicts of results.
|
|
714
648
|
'''
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
return
|
|
719
|
-
|
|
720
|
-
link00, sock00 = await s_link.linksock()
|
|
721
|
-
|
|
722
|
-
try:
|
|
723
|
-
async with link00:
|
|
724
|
-
|
|
725
|
-
todo = s_common.todo(_spawn_scrape, sock00, text, form=form, refang=refang, first=first)
|
|
726
|
-
link00.schedCoro(s_coro.spawn(todo, log_conf=s_common._getLogConfFromEnv()))
|
|
727
|
-
|
|
728
|
-
while (mesg := await link00.rx()) is not None:
|
|
729
|
-
|
|
730
|
-
info = s_common.result(mesg)
|
|
731
|
-
if info is None:
|
|
732
|
-
return
|
|
733
|
-
|
|
734
|
-
yield info
|
|
735
|
-
|
|
736
|
-
finally:
|
|
737
|
-
sock00.close()
|
|
649
|
+
matches = await s_coro.semafork(_contextScrapeList, text, form=form, refang=refang, first=first)
|
|
650
|
+
for info in matches:
|
|
651
|
+
yield info
|
|
738
652
|
|
|
739
653
|
async def scrapeAsync(text, ptype=None, refang=True, first=False):
|
|
740
654
|
'''
|
|
741
|
-
Scrape types from a blob of text and return node tuples,
|
|
655
|
+
Scrape types from a blob of text and return node tuples, using the shared forked process pool.
|
|
742
656
|
|
|
743
657
|
Args:
|
|
744
658
|
text (str): Text to scrape.
|
|
@@ -749,6 +663,6 @@ async def scrapeAsync(text, ptype=None, refang=True, first=False):
|
|
|
749
663
|
Returns:
|
|
750
664
|
(str, object): Yield tuples of node ndef values.
|
|
751
665
|
'''
|
|
752
|
-
|
|
753
|
-
|
|
666
|
+
matches = await s_coro.semafork(_contextScrapeList, text, form=ptype, refang=refang, first=first)
|
|
667
|
+
for info in matches:
|
|
754
668
|
yield info.get('form'), info.get('valu')
|
synapse/lib/storm.py
CHANGED
|
@@ -1357,30 +1357,42 @@ stormcmds = (
|
|
|
1357
1357
|
'descr': "List existing cron jobs in the cortex.",
|
|
1358
1358
|
'cmdargs': (),
|
|
1359
1359
|
'storm': '''
|
|
1360
|
+
init {
|
|
1361
|
+
$conf = ({
|
|
1362
|
+
"columns": [
|
|
1363
|
+
{"name": "user", "width": 24},
|
|
1364
|
+
{"name": "iden", "width": 10},
|
|
1365
|
+
{"name": "view", "width": 10},
|
|
1366
|
+
{"name": "en?", "width": 3},
|
|
1367
|
+
{"name": "rpt?", "width": 4},
|
|
1368
|
+
{"name": "now?", "width": 4},
|
|
1369
|
+
{"name": "err?", "width": 4},
|
|
1370
|
+
{"name": "# start", "width": 7},
|
|
1371
|
+
{"name": "last start", "width": 16},
|
|
1372
|
+
{"name": "last end", "width": 16},
|
|
1373
|
+
{"name": "query", "newlines": "split"},
|
|
1374
|
+
],
|
|
1375
|
+
"separators": {
|
|
1376
|
+
"row:outline": false,
|
|
1377
|
+
"column:outline": false,
|
|
1378
|
+
"header:row": "#",
|
|
1379
|
+
"data:row": "",
|
|
1380
|
+
"column": "",
|
|
1381
|
+
},
|
|
1382
|
+
})
|
|
1383
|
+
$printer = $lib.tabular.printer($conf)
|
|
1384
|
+
}
|
|
1360
1385
|
$crons = $lib.cron.list()
|
|
1361
|
-
|
|
1362
1386
|
if $crons {
|
|
1363
|
-
$lib.print(
|
|
1364
|
-
|
|
1387
|
+
$lib.print($printer.header())
|
|
1365
1388
|
for $cron in $crons {
|
|
1366
|
-
|
|
1367
1389
|
$job = $cron.pprint()
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
$
|
|
1374
|
-
$isrunning = $job.isrunning.ljust(4)
|
|
1375
|
-
$iserr = $job.iserr.ljust(4)
|
|
1376
|
-
$startcount = $lib.str.format("{startcount}", startcount=$job.startcount).ljust(7)
|
|
1377
|
-
$laststart = $job.laststart.ljust(16)
|
|
1378
|
-
$lastend = $job.lastend.ljust(16)
|
|
1379
|
-
|
|
1380
|
-
$lib.print("{user} {iden} {view} {enabled} {isrecur} {isrunning} {iserr} {startcount} {laststart} {lastend} {query}",
|
|
1381
|
-
user=$user, iden=$iden, view=$view, enabled=$enabled, isrecur=$isrecur,
|
|
1382
|
-
isrunning=$isrunning, iserr=$iserr, startcount=$startcount,
|
|
1383
|
-
laststart=$laststart, lastend=$lastend, query=$job.query)
|
|
1390
|
+
$row = (
|
|
1391
|
+
$job.user, $job.idenshort, $job.viewshort, $job.enabled,
|
|
1392
|
+
$job.isrecur, $job.isrunning, $job.iserr, `{$job.startcount}`,
|
|
1393
|
+
$job.laststart, $job.lastend, $job.query
|
|
1394
|
+
)
|
|
1395
|
+
$lib.print($printer.row($row))
|
|
1384
1396
|
}
|
|
1385
1397
|
} else {
|
|
1386
1398
|
$lib.print("No cron jobs found")
|
|
@@ -5268,6 +5280,13 @@ class BackgroundCmd(Cmd):
|
|
|
5268
5280
|
async for item in genr:
|
|
5269
5281
|
yield item
|
|
5270
5282
|
|
|
5283
|
+
_query = await s_stormtypes.tostr(self.opts.query)
|
|
5284
|
+
query = await runt.getStormQuery(_query)
|
|
5285
|
+
|
|
5286
|
+
# make sure the subquery *could* have run
|
|
5287
|
+
async with runt.getSubRuntime(query) as subr:
|
|
5288
|
+
query.validate(subr)
|
|
5289
|
+
|
|
5271
5290
|
runtprims = await s_stormtypes.toprim(self.runt.getScopeVars(), use_list=True)
|
|
5272
5291
|
runtvars = {k: v for (k, v) in runtprims.items() if s_msgpack.isok(v)}
|
|
5273
5292
|
|
|
@@ -5277,12 +5296,6 @@ class BackgroundCmd(Cmd):
|
|
|
5277
5296
|
'vars': runtvars,
|
|
5278
5297
|
}
|
|
5279
5298
|
|
|
5280
|
-
_query = await s_stormtypes.tostr(self.opts.query)
|
|
5281
|
-
query = await runt.getStormQuery(_query)
|
|
5282
|
-
|
|
5283
|
-
# make sure the subquery *could* have run with existing vars
|
|
5284
|
-
query.validate(runt)
|
|
5285
|
-
|
|
5286
5299
|
coro = self.execStormTask(query, opts)
|
|
5287
5300
|
runt.snap.core.schedCoro(coro)
|
|
5288
5301
|
|
|
@@ -5339,9 +5352,12 @@ class ParallelCmd(Cmd):
|
|
|
5339
5352
|
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
5340
5353
|
|
|
5341
5354
|
size = await s_stormtypes.toint(self.opts.size)
|
|
5342
|
-
query = await runt.getStormQuery(self.opts.query)
|
|
5343
5355
|
|
|
5344
|
-
|
|
5356
|
+
_query = await s_stormtypes.tostr(self.opts.query)
|
|
5357
|
+
query = await runt.getStormQuery(_query)
|
|
5358
|
+
|
|
5359
|
+
async with runt.getSubRuntime(query) as subr:
|
|
5360
|
+
query.validate(subr)
|
|
5345
5361
|
|
|
5346
5362
|
async with await s_base.Base.anit() as base:
|
|
5347
5363
|
|
synapse/lib/stormlib/modelext.py
CHANGED
|
@@ -105,6 +105,21 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
105
105
|
'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
|
|
106
106
|
),
|
|
107
107
|
'returns': {'type': 'null'}}},
|
|
108
|
+
{'name': 'addType', 'desc': 'Add an extended type definition to the data model.',
|
|
109
|
+
'type': {'type': 'function', '_funcname': 'addType',
|
|
110
|
+
'args': (
|
|
111
|
+
{'name': 'typename', 'type': 'str', 'desc': 'The name of the type to add.'},
|
|
112
|
+
{'name': 'basetype', 'type': 'str', 'desc': 'The base type the type is derived from.'},
|
|
113
|
+
{'name': 'typeopts', 'type': 'dict', 'desc': 'A Synapse type opts dictionary.'},
|
|
114
|
+
{'name': 'typeinfo', 'type': 'dict', 'desc': 'A Synapse type info dictionary.'},
|
|
115
|
+
),
|
|
116
|
+
'returns': {'type': 'null'}}},
|
|
117
|
+
{'name': 'delType', 'desc': 'Remove an extended type definition from the model.',
|
|
118
|
+
'type': {'type': 'function', '_funcname': 'delType',
|
|
119
|
+
'args': (
|
|
120
|
+
{'name': 'typename', 'type': 'str', 'desc': 'The extended type to remove.'},
|
|
121
|
+
),
|
|
122
|
+
'returns': {'type': 'null'}}},
|
|
108
123
|
)
|
|
109
124
|
_storm_lib_path = ('model', 'ext')
|
|
110
125
|
|
|
@@ -122,6 +137,8 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
122
137
|
'addExtModel': self.addExtModel,
|
|
123
138
|
'addEdge': self.addEdge,
|
|
124
139
|
'delEdge': self.delEdge,
|
|
140
|
+
'addType': self.addType,
|
|
141
|
+
'delType': self.delType,
|
|
125
142
|
}
|
|
126
143
|
|
|
127
144
|
# TODO type docs in the new convention
|
|
@@ -210,6 +227,7 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
210
227
|
return await self.runt.snap.core.getExtModel()
|
|
211
228
|
|
|
212
229
|
async def addExtModel(self, model):
|
|
230
|
+
self.runt.reqAdmin()
|
|
213
231
|
model = await s_stormtypes.toprim(model)
|
|
214
232
|
return await self.runt.snap.core.addExtModel(model)
|
|
215
233
|
|
|
@@ -249,3 +267,16 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
249
267
|
|
|
250
268
|
s_stormtypes.confirm(('model', 'edge', 'del'))
|
|
251
269
|
await self.runt.snap.core.delEdge((n1form, verb, n2form))
|
|
270
|
+
|
|
271
|
+
async def addType(self, typename, basetype, typeopts, typeinfo):
|
|
272
|
+
typename = await s_stormtypes.tostr(typename)
|
|
273
|
+
basetype = await s_stormtypes.tostr(basetype)
|
|
274
|
+
typeopts = await s_stormtypes.toprim(typeopts)
|
|
275
|
+
typeinfo = await s_stormtypes.toprim(typeinfo)
|
|
276
|
+
s_stormtypes.confirm(('model', 'type', 'add', typename))
|
|
277
|
+
await self.runt.snap.core.addType(typename, basetype, typeopts, typeinfo)
|
|
278
|
+
|
|
279
|
+
async def delType(self, typename):
|
|
280
|
+
typename = await s_stormtypes.tostr(typename)
|
|
281
|
+
s_stormtypes.confirm(('model', 'type', 'del', typename))
|
|
282
|
+
await self.runt.snap.core.delType(typename)
|
synapse/lib/stormlib/scrape.py
CHANGED
|
@@ -155,10 +155,7 @@ class LibScrape(s_stormtypes.Lib):
|
|
|
155
155
|
if fangs:
|
|
156
156
|
_fangs = {src: dst for (src, dst) in fangs}
|
|
157
157
|
_fangre = s_scrape.genFangRegex(_fangs)
|
|
158
|
-
|
|
159
|
-
scrape_text, offsets = s_scrape.refang_text2(text, re=_fangre, fangs=_fangs)
|
|
160
|
-
else:
|
|
161
|
-
scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
|
|
158
|
+
scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
|
|
162
159
|
|
|
163
160
|
async for info in s_scrape.genMatchesAsync(scrape_text, regx, opts=opts):
|
|
164
161
|
valu = info.pop('valu')
|
synapse/lib/stormtypes.py
CHANGED
|
@@ -7545,6 +7545,9 @@ class View(Prim):
|
|
|
7545
7545
|
{'name': 'parent', 'desc': 'The parent View. Will be ``$lib.null`` if the view is not a fork.', 'type': 'str'},
|
|
7546
7546
|
{'name': 'triggers', 'desc': 'The ``trigger`` objects associated with the ``view``.',
|
|
7547
7547
|
'type': 'list', },
|
|
7548
|
+
{'name': 'children', 'desc': 'Yield Views which are children of this View.',
|
|
7549
|
+
'type': {'type': 'function', '_funcname': '_methGetChildren',
|
|
7550
|
+
'returns': {'name': 'yields', 'type': 'view', 'desc': 'Child Views.', }}},
|
|
7548
7551
|
{'name': 'set', 'desc': '''
|
|
7549
7552
|
Set a view configuration option.
|
|
7550
7553
|
|
|
@@ -7832,6 +7835,7 @@ class View(Prim):
|
|
|
7832
7835
|
'merge': self._methViewMerge,
|
|
7833
7836
|
'detach': self.detach,
|
|
7834
7837
|
'addNode': self.addNode,
|
|
7838
|
+
'children': self._methGetChildren,
|
|
7835
7839
|
'getEdges': self._methGetEdges,
|
|
7836
7840
|
'wipeLayer': self._methWipeLayer,
|
|
7837
7841
|
'swapLayer': self._methSwapLayer,
|
|
@@ -7967,6 +7971,12 @@ class View(Prim):
|
|
|
7967
7971
|
async for valu in view.iterPropValues(propname):
|
|
7968
7972
|
yield valu
|
|
7969
7973
|
|
|
7974
|
+
@stormfunc(readonly=True)
|
|
7975
|
+
async def _methGetChildren(self):
|
|
7976
|
+
view = self._reqView()
|
|
7977
|
+
async for child in view.children():
|
|
7978
|
+
yield View(self.runt, await child.pack(), path=self.path)
|
|
7979
|
+
|
|
7970
7980
|
@stormfunc(readonly=True)
|
|
7971
7981
|
async def _methGetEdges(self, verb=None):
|
|
7972
7982
|
verb = await toprim(verb)
|
|
@@ -9610,7 +9620,7 @@ def fromprim(valu, path=None, basetypes=True):
|
|
|
9610
9620
|
|
|
9611
9621
|
return valu
|
|
9612
9622
|
|
|
9613
|
-
async def tostor(valu):
|
|
9623
|
+
async def tostor(valu, isndef=False):
|
|
9614
9624
|
|
|
9615
9625
|
if isinstance(valu, Number):
|
|
9616
9626
|
return str(valu.value())
|
|
@@ -9633,6 +9643,9 @@ async def tostor(valu):
|
|
|
9633
9643
|
pass
|
|
9634
9644
|
return retn
|
|
9635
9645
|
|
|
9646
|
+
if isndef and isinstance(valu, s_node.Node):
|
|
9647
|
+
return valu.ndef
|
|
9648
|
+
|
|
9636
9649
|
return await toprim(valu)
|
|
9637
9650
|
|
|
9638
9651
|
async def tocmprvalu(valu):
|
|
@@ -9676,6 +9689,9 @@ async def tostr(valu, noneok=False):
|
|
|
9676
9689
|
if isinstance(valu, bytes):
|
|
9677
9690
|
return valu.decode('utf8', 'surrogatepass')
|
|
9678
9691
|
|
|
9692
|
+
if isinstance(valu, s_node.Node):
|
|
9693
|
+
return valu.repr()
|
|
9694
|
+
|
|
9679
9695
|
return str(valu)
|
|
9680
9696
|
except Exception as e:
|
|
9681
9697
|
mesg = f'Failed to make a string from {valu!r}.'
|
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 188, 1)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = '2cf97aa2a3266442393dfc78525749d3f65d9fc7'
|
synapse/lib/view.py
CHANGED
|
@@ -1315,6 +1315,14 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1315
1315
|
|
|
1316
1316
|
todo.append(child)
|
|
1317
1317
|
|
|
1318
|
+
async def children(self):
|
|
1319
|
+
for view in list(self.core.views.values()):
|
|
1320
|
+
if view.parent != self:
|
|
1321
|
+
await asyncio.sleep(0)
|
|
1322
|
+
continue
|
|
1323
|
+
|
|
1324
|
+
yield view
|
|
1325
|
+
|
|
1318
1326
|
async def insertParentFork(self, useriden, name=None):
|
|
1319
1327
|
'''
|
|
1320
1328
|
Insert a new View between a forked View and its parent.
|
|
@@ -1614,9 +1622,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1614
1622
|
if trig is not None:
|
|
1615
1623
|
return self.triggers.get(tdef['iden']).pack()
|
|
1616
1624
|
|
|
1617
|
-
|
|
1618
|
-
if gate is not None:
|
|
1619
|
-
raise s_exc.DupIden(mesg='An AuthGate with this iden already exists')
|
|
1625
|
+
self.core.auth.reqNoAuthGate(tdef['iden'])
|
|
1620
1626
|
|
|
1621
1627
|
user = self.core.auth.user(tdef['user'])
|
|
1622
1628
|
await self.core.getStormQuery(tdef['storm'])
|
synapse/models/base.py
CHANGED
|
@@ -107,6 +107,19 @@ class BaseModule(s_module.CoreModule):
|
|
|
107
107
|
|
|
108
108
|
('meta:sophistication', ('int', {'enums': sophenums}), {
|
|
109
109
|
'doc': 'A sophistication score with named values: very low, low, medium, high, and very high.'}),
|
|
110
|
+
|
|
111
|
+
('meta:aggregate:type:taxonomy', ('taxonomy', {}), {
|
|
112
|
+
'interfaces': ('meta:taxonomy',),
|
|
113
|
+
'doc': 'A type of item being counted in aggregate.'}),
|
|
114
|
+
|
|
115
|
+
('meta:aggregate', ('guid', {}), {
|
|
116
|
+
'display': {
|
|
117
|
+
'columns': (
|
|
118
|
+
{'type': 'prop', 'opts': {'name': 'type'}},
|
|
119
|
+
{'type': 'prop', 'opts': {'name': 'count'}},
|
|
120
|
+
),
|
|
121
|
+
},
|
|
122
|
+
'doc': 'A node which represents an aggregate count of a specific type.'}),
|
|
110
123
|
),
|
|
111
124
|
'interfaces': (
|
|
112
125
|
('meta:taxonomy', {
|
|
@@ -285,6 +298,20 @@ class BaseModule(s_module.CoreModule):
|
|
|
285
298
|
'doc': 'An external identifier for the rule.'}),
|
|
286
299
|
)),
|
|
287
300
|
|
|
301
|
+
('meta:aggregate:type:taxonomy', {}, ()),
|
|
302
|
+
('meta:aggregate', {}, (
|
|
303
|
+
|
|
304
|
+
('type', ('meta:aggregate:type:taxonomy', {}), {
|
|
305
|
+
'ex': 'casualties.civilian',
|
|
306
|
+
'doc': 'The type of items being counted in aggregate.'}),
|
|
307
|
+
|
|
308
|
+
('time', ('time', {}), {
|
|
309
|
+
'doc': 'The time that the count was computed.'}),
|
|
310
|
+
|
|
311
|
+
('count', ('int', {}), {
|
|
312
|
+
'doc': 'The number of items counted in aggregate.'}),
|
|
313
|
+
)),
|
|
314
|
+
|
|
288
315
|
('graph:cluster', {}, (
|
|
289
316
|
('name', ('str', {'lower': True}), {
|
|
290
317
|
'doc': 'A human friendly name for the cluster.'}),
|
synapse/models/files.py
CHANGED
|
@@ -284,6 +284,16 @@ class FileModule(s_module.CoreModule):
|
|
|
284
284
|
'doc': 'A parent file that fully contains the specified child file.',
|
|
285
285
|
}),
|
|
286
286
|
|
|
287
|
+
('file:attachment', ('guid', {}), {
|
|
288
|
+
'display': {
|
|
289
|
+
'columns': (
|
|
290
|
+
{'type': 'prop', 'opts': {'name': 'name'}},
|
|
291
|
+
{'type': 'prop', 'opts': {'name': 'file'}},
|
|
292
|
+
{'type': 'prop', 'opts': {'name': 'text'}},
|
|
293
|
+
),
|
|
294
|
+
},
|
|
295
|
+
'doc': 'A file attachment.'}),
|
|
296
|
+
|
|
287
297
|
('file:archive:entry', ('guid', {}), {
|
|
288
298
|
'doc': 'An archive entry representing a file and metadata within a parent archive file.'}),
|
|
289
299
|
|
|
@@ -601,6 +611,18 @@ class FileModule(s_module.CoreModule):
|
|
|
601
611
|
}),
|
|
602
612
|
)),
|
|
603
613
|
|
|
614
|
+
('file:attachment', {}, (
|
|
615
|
+
|
|
616
|
+
('name', ('file:path', {}), {
|
|
617
|
+
'doc': 'The name of the attached file.'}),
|
|
618
|
+
|
|
619
|
+
('text', ('str', {}), {
|
|
620
|
+
'doc': 'Any text associated with the file such as alt-text for images.'}),
|
|
621
|
+
|
|
622
|
+
('file', ('file:bytes', {}), {
|
|
623
|
+
'doc': 'The file which was attached.'}),
|
|
624
|
+
)),
|
|
625
|
+
|
|
604
626
|
('file:archive:entry', {}, (
|
|
605
627
|
|
|
606
628
|
('parent', ('file:bytes', {}), {
|
synapse/models/inet.py
CHANGED
|
@@ -1369,6 +1369,9 @@ class InetModule(s_module.CoreModule):
|
|
|
1369
1369
|
'doc': 'A username string.'
|
|
1370
1370
|
}),
|
|
1371
1371
|
|
|
1372
|
+
('inet:service:object', ('ndef', {'interfaces': ('inet:service:object',)}), {
|
|
1373
|
+
'doc': 'An ndef type including all forms which implement the inet:service:object interface.'}),
|
|
1374
|
+
|
|
1372
1375
|
('inet:search:query', ('guid', {}), {
|
|
1373
1376
|
'interfaces': ('inet:service:action',),
|
|
1374
1377
|
'doc': 'An instance of a search query issued to a search engine.',
|
|
@@ -1530,6 +1533,14 @@ class InetModule(s_module.CoreModule):
|
|
|
1530
1533
|
'interfaces': ('inet:service:object',),
|
|
1531
1534
|
'doc': 'An account within a service platform. Accounts may be instance specific.'}),
|
|
1532
1535
|
|
|
1536
|
+
('inet:service:relationship:type:taxonomy', ('taxonomy', {}), {
|
|
1537
|
+
'interfaces': ('meta:taxonomy',),
|
|
1538
|
+
'doc': 'A service object relationship type taxonomy.'}),
|
|
1539
|
+
|
|
1540
|
+
('inet:service:relationship', ('guid', {}), {
|
|
1541
|
+
'interfaces': ('inet:service:object',),
|
|
1542
|
+
'doc': 'A relationship between two service objects.'}),
|
|
1543
|
+
|
|
1533
1544
|
('inet:service:permission:type:taxonomy', ('taxonomy', {}), {
|
|
1534
1545
|
'interfaces': ('meta:taxonomy',),
|
|
1535
1546
|
'doc': 'A permission type taxonomy.'}),
|
|
@@ -1588,6 +1599,10 @@ class InetModule(s_module.CoreModule):
|
|
|
1588
1599
|
'interfaces': ('meta:taxonomy',),
|
|
1589
1600
|
'doc': 'A message type taxonomy.'}),
|
|
1590
1601
|
|
|
1602
|
+
('inet:service:emote', ('guid', {}), {
|
|
1603
|
+
'interfaces': ('inet:service:object',),
|
|
1604
|
+
'doc': 'An emote or reaction by an account.'}),
|
|
1605
|
+
|
|
1591
1606
|
('inet:service:access', ('guid', {}), {
|
|
1592
1607
|
'interfaces': ('inet:service:action',),
|
|
1593
1608
|
'doc': 'Represents a user access request to a service resource.'}),
|
|
@@ -2015,8 +2030,11 @@ class InetModule(s_module.CoreModule):
|
|
|
2015
2030
|
'doc': 'The guid of the destination process.'
|
|
2016
2031
|
}),
|
|
2017
2032
|
('dst:exe', ('file:bytes', {}), {
|
|
2018
|
-
'doc': 'The file (executable) that received the connection.'
|
|
2019
|
-
|
|
2033
|
+
'doc': 'The file (executable) that received the connection.'}),
|
|
2034
|
+
|
|
2035
|
+
('dst:txfiles', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
|
|
2036
|
+
'doc': 'An array of files sent by the destination host.'}),
|
|
2037
|
+
|
|
2020
2038
|
('dst:txcount', ('int', {}), {
|
|
2021
2039
|
'doc': 'The number of packets sent by the destination host.'
|
|
2022
2040
|
}),
|
|
@@ -2049,8 +2067,11 @@ class InetModule(s_module.CoreModule):
|
|
|
2049
2067
|
'doc': 'The guid of the source process.'
|
|
2050
2068
|
}),
|
|
2051
2069
|
('src:exe', ('file:bytes', {}), {
|
|
2052
|
-
'doc': 'The file (executable) that created the connection.'
|
|
2053
|
-
|
|
2070
|
+
'doc': 'The file (executable) that created the connection.'}),
|
|
2071
|
+
|
|
2072
|
+
('src:txfiles', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
|
|
2073
|
+
'doc': 'An array of files sent by the source host.'}),
|
|
2074
|
+
|
|
2054
2075
|
('src:txcount', ('int', {}), {
|
|
2055
2076
|
'doc': 'The number of packets sent by the source host.'
|
|
2056
2077
|
}),
|
|
@@ -3586,6 +3607,20 @@ class InetModule(s_module.CoreModule):
|
|
|
3586
3607
|
'doc': 'Current profile details associated with the account.'}),
|
|
3587
3608
|
)),
|
|
3588
3609
|
|
|
3610
|
+
('inet:service:relationship:type:taxonomy', {}, ()),
|
|
3611
|
+
('inet:service:relationship', {}, (
|
|
3612
|
+
|
|
3613
|
+
('source', ('inet:service:object', {}), {
|
|
3614
|
+
'doc': 'The source object.'}),
|
|
3615
|
+
|
|
3616
|
+
('target', ('inet:service:object', {}), {
|
|
3617
|
+
'doc': 'The target object.'}),
|
|
3618
|
+
|
|
3619
|
+
('type', ('inet:service:relationship:type:taxonomy', {}), {
|
|
3620
|
+
'ex': 'follows',
|
|
3621
|
+
'doc': 'The type of relationship between the source and the target.'}),
|
|
3622
|
+
)),
|
|
3623
|
+
|
|
3589
3624
|
('inet:service:group', {}, ( # inet:service:object
|
|
3590
3625
|
|
|
3591
3626
|
('id', ('str', {'strip': True}), {
|
|
@@ -3743,6 +3778,16 @@ class InetModule(s_module.CoreModule):
|
|
|
3743
3778
|
'doc': 'The file which was attached to the message.'}),
|
|
3744
3779
|
)),
|
|
3745
3780
|
|
|
3781
|
+
('inet:service:emote', {}, (
|
|
3782
|
+
|
|
3783
|
+
('about', ('inet:service:object', {}), {
|
|
3784
|
+
'doc': 'The node that the emote is about.'}),
|
|
3785
|
+
|
|
3786
|
+
('text', ('str', {'strip': True}), {
|
|
3787
|
+
'ex': ':partyparrot:',
|
|
3788
|
+
'doc': 'The unicode or emote text of the reaction.'}),
|
|
3789
|
+
)),
|
|
3790
|
+
|
|
3746
3791
|
('inet:service:channel', {}, (
|
|
3747
3792
|
|
|
3748
3793
|
('name', ('str', {'onespace': True, 'lower': True}), {
|