synapse 2.187.0__py311-none-any.whl → 2.188.1__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (46) hide show
  1. synapse/cortex.py +131 -7
  2. synapse/datamodel.py +20 -4
  3. synapse/exc.py +14 -1
  4. synapse/lib/ast.py +6 -4
  5. synapse/lib/auth.py +9 -0
  6. synapse/lib/drive.py +1 -1
  7. synapse/lib/httpapi.py +2 -1
  8. synapse/lib/nexus.py +6 -0
  9. synapse/lib/node.py +5 -3
  10. synapse/lib/scrape.py +18 -104
  11. synapse/lib/storm.py +44 -28
  12. synapse/lib/stormlib/modelext.py +31 -0
  13. synapse/lib/stormlib/scrape.py +1 -4
  14. synapse/lib/stormtypes.py +17 -1
  15. synapse/lib/version.py +2 -2
  16. synapse/lib/view.py +9 -3
  17. synapse/models/base.py +27 -0
  18. synapse/models/files.py +22 -0
  19. synapse/models/inet.py +49 -4
  20. synapse/models/orgs.py +64 -2
  21. synapse/models/proj.py +1 -6
  22. synapse/models/risk.py +65 -0
  23. synapse/tests/test_cortex.py +21 -0
  24. synapse/tests/test_lib_agenda.py +13 -0
  25. synapse/tests/test_lib_auth.py +15 -0
  26. synapse/tests/test_lib_cell.py +170 -159
  27. synapse/tests/test_lib_httpapi.py +6 -0
  28. synapse/tests/test_lib_nexus.py +26 -0
  29. synapse/tests/test_lib_scrape.py +14 -6
  30. synapse/tests/test_lib_storm.py +48 -0
  31. synapse/tests/test_lib_stormlib_modelext.py +76 -1
  32. synapse/tests/test_lib_stormlib_scrape.py +0 -8
  33. synapse/tests/test_lib_stormtypes.py +1 -1
  34. synapse/tests/test_lib_trigger.py +8 -0
  35. synapse/tests/test_lib_view.py +24 -0
  36. synapse/tests/test_model_base.py +11 -0
  37. synapse/tests/test_model_files.py +19 -0
  38. synapse/tests/test_model_inet.py +33 -0
  39. synapse/tests/test_model_orgs.py +39 -0
  40. synapse/tests/test_model_proj.py +11 -1
  41. synapse/tests/test_model_risk.py +32 -0
  42. {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/METADATA +1 -1
  43. {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/RECORD +46 -46
  44. {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/WHEEL +1 -1
  45. {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/LICENSE +0 -0
  46. {synapse-2.187.0.dist-info → synapse-2.188.1.dist-info}/top_level.txt +0 -0
synapse/lib/scrape.py CHANGED
@@ -24,8 +24,6 @@ ipaddress = s_common.ipaddress
24
24
 
25
25
  logger = logging.getLogger(__name__)
26
26
 
27
- SCRAPE_SPAWN_LENGTH = 5000
28
-
29
27
  tldlist = list(s_data.get('iana.tlds'))
30
28
  tldlist.extend([
31
29
  'bit',
@@ -445,6 +443,9 @@ def _rewriteRawValu(text: str, offsets: dict, info: dict):
445
443
  info['match'] = match
446
444
  info['offset'] = baseoff + offset
447
445
 
446
+ def _genMatchList(text: str, regx: regex.Regex, opts: dict):
447
+ return [info for info in _genMatches(text, regx, opts)]
448
+
448
449
  def _genMatches(text: str, regx: regex.Regex, opts: dict):
449
450
 
450
451
  cb = opts.get('callback')
@@ -504,23 +505,9 @@ def genMatches(text: str, regx: regex.Regex, opts: dict):
504
505
  for match in _genMatches(text, regx, opts):
505
506
  yield match
506
507
 
507
- def _spawn_genmatches(sock, text, regx, opts): # pragma: no cover
508
- '''
509
- Multiprocessing target for generating matches.
510
- '''
511
- try:
512
- for info in _genMatches(text, regx, opts):
513
- sock.sendall(s_msgpack.en((True, info)))
514
-
515
- sock.sendall(s_msgpack.en((True, None)))
516
-
517
- except Exception as e:
518
- mesg = s_common.retnexc(e)
519
- sock.sendall(s_msgpack.en(mesg))
520
-
521
508
  async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
522
509
  '''
523
- Generate regular expression matches for a blob of text, potentially in a spawned process.
510
+ Generate regular expression matches for a blob of text, using the shared forked process pool.
524
511
 
525
512
  Args:
526
513
  text (str): The text to generate matches for.
@@ -547,30 +534,9 @@ async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
547
534
  Yields:
548
535
  dict: A dictionary of match results.
549
536
  '''
550
- if len(text) < SCRAPE_SPAWN_LENGTH:
551
- for match in _genMatches(text, regx, opts):
552
- yield match
553
- return
554
-
555
- link00, sock00 = await s_link.linksock()
556
-
557
- try:
558
- async with link00:
559
-
560
- todo = s_common.todo(_spawn_genmatches, sock00, text, regx, opts)
561
- link00.schedCoro(s_coro.spawn(todo, log_conf=s_common._getLogConfFromEnv()))
562
-
563
- while (mesg := await link00.rx()) is not None:
564
-
565
- info = s_common.result(mesg)
566
- if info is None:
567
- return
568
-
569
- yield info
570
-
571
- finally:
572
- sock00.close()
573
-
537
+ matches = await s_coro.semafork(_genMatchList, text, regx, opts)
538
+ for info in matches:
539
+ yield info
574
540
 
575
541
  def _contextMatches(scrape_text, text, ruletype, refang, offsets):
576
542
 
@@ -585,33 +551,16 @@ def _contextMatches(scrape_text, text, ruletype, refang, offsets):
585
551
 
586
552
  yield info
587
553
 
588
- def _contextScrape(text, form=None, refang=True, first=False):
589
- scrape_text = text
590
- offsets = {}
591
- if refang:
592
- scrape_text, offsets = refang_text2(text)
593
-
594
- for ruletype, blobs in _regexes.items():
595
- if form and form != ruletype:
596
- continue
554
+ def _contextScrapeList(text, form=None, refang=True, first=False):
555
+ return [info for info in _contextScrape(text, form=form, refang=refang, first=first)]
597
556
 
598
- for info in _contextMatches(scrape_text, text, ruletype, refang, offsets):
599
-
600
- yield info
601
-
602
- if first:
603
- return
604
-
605
- async def _contextScrapeAsync(text, form=None, refang=True, first=False):
557
+ def _contextScrape(text, form=None, refang=True, first=False):
606
558
  scrape_text = text
607
559
  offsets = {}
608
560
  if refang:
609
561
  scrape_text, offsets = refang_text2(text)
610
562
 
611
563
  for ruletype, blobs in _regexes.items():
612
-
613
- await asyncio.sleep(0)
614
-
615
564
  if form and form != ruletype:
616
565
  continue
617
566
 
@@ -666,27 +615,12 @@ def scrape(text, ptype=None, refang=True, first=False):
666
615
  Returns:
667
616
  (str, object): Yield tuples of node ndef values.
668
617
  '''
669
-
670
- for info in contextScrape(text, form=ptype, refang=refang, first=first):
618
+ for info in _contextScrape(text, form=ptype, refang=refang, first=first):
671
619
  yield info.get('form'), info.get('valu')
672
620
 
673
- def _spawn_scrape(sock, text, form=None, refang=True, first=False): # pragma: no cover
674
- '''
675
- Multiprocessing target for scraping text.
676
- '''
677
- try:
678
- for info in _contextScrape(text, form=form, refang=refang, first=first):
679
- sock.sendall(s_msgpack.en((True, info)))
680
-
681
- sock.sendall(s_msgpack.en((True, None)))
682
-
683
- except Exception as e:
684
- mesg = s_common.retnexc(e)
685
- sock.sendall(s_msgpack.en(mesg))
686
-
687
621
  async def contextScrapeAsync(text, form=None, refang=True, first=False):
688
622
  '''
689
- Scrape types from a blob of text and yield info dictionaries, potentially in a spawned process.
623
+ Scrape types from a blob of text and yield info dictionaries, using the shared forked process pool.
690
624
 
691
625
  Args:
692
626
  text (str): Text to scrape.
@@ -712,33 +646,13 @@ async def contextScrapeAsync(text, form=None, refang=True, first=False):
712
646
  Returns:
713
647
  (dict): Yield info dicts of results.
714
648
  '''
715
- if len(text) < SCRAPE_SPAWN_LENGTH:
716
- async for info in _contextScrapeAsync(text, form=form, refang=refang, first=first):
717
- yield info
718
- return
719
-
720
- link00, sock00 = await s_link.linksock()
721
-
722
- try:
723
- async with link00:
724
-
725
- todo = s_common.todo(_spawn_scrape, sock00, text, form=form, refang=refang, first=first)
726
- link00.schedCoro(s_coro.spawn(todo, log_conf=s_common._getLogConfFromEnv()))
727
-
728
- while (mesg := await link00.rx()) is not None:
729
-
730
- info = s_common.result(mesg)
731
- if info is None:
732
- return
733
-
734
- yield info
735
-
736
- finally:
737
- sock00.close()
649
+ matches = await s_coro.semafork(_contextScrapeList, text, form=form, refang=refang, first=first)
650
+ for info in matches:
651
+ yield info
738
652
 
739
653
  async def scrapeAsync(text, ptype=None, refang=True, first=False):
740
654
  '''
741
- Scrape types from a blob of text and return node tuples, potentially in a spawned process.
655
+ Scrape types from a blob of text and return node tuples, using the shared forked process pool.
742
656
 
743
657
  Args:
744
658
  text (str): Text to scrape.
@@ -749,6 +663,6 @@ async def scrapeAsync(text, ptype=None, refang=True, first=False):
749
663
  Returns:
750
664
  (str, object): Yield tuples of node ndef values.
751
665
  '''
752
-
753
- async for info in contextScrapeAsync(text, form=ptype, refang=refang, first=first):
666
+ matches = await s_coro.semafork(_contextScrapeList, text, form=ptype, refang=refang, first=first)
667
+ for info in matches:
754
668
  yield info.get('form'), info.get('valu')
synapse/lib/storm.py CHANGED
@@ -1357,30 +1357,42 @@ stormcmds = (
1357
1357
  'descr': "List existing cron jobs in the cortex.",
1358
1358
  'cmdargs': (),
1359
1359
  'storm': '''
1360
+ init {
1361
+ $conf = ({
1362
+ "columns": [
1363
+ {"name": "user", "width": 24},
1364
+ {"name": "iden", "width": 10},
1365
+ {"name": "view", "width": 10},
1366
+ {"name": "en?", "width": 3},
1367
+ {"name": "rpt?", "width": 4},
1368
+ {"name": "now?", "width": 4},
1369
+ {"name": "err?", "width": 4},
1370
+ {"name": "# start", "width": 7},
1371
+ {"name": "last start", "width": 16},
1372
+ {"name": "last end", "width": 16},
1373
+ {"name": "query", "newlines": "split"},
1374
+ ],
1375
+ "separators": {
1376
+ "row:outline": false,
1377
+ "column:outline": false,
1378
+ "header:row": "#",
1379
+ "data:row": "",
1380
+ "column": "",
1381
+ },
1382
+ })
1383
+ $printer = $lib.tabular.printer($conf)
1384
+ }
1360
1385
  $crons = $lib.cron.list()
1361
-
1362
1386
  if $crons {
1363
- $lib.print("user iden view en? rpt? now? err? # start last start last end query")
1364
-
1387
+ $lib.print($printer.header())
1365
1388
  for $cron in $crons {
1366
-
1367
1389
  $job = $cron.pprint()
1368
-
1369
- $user = $job.user.ljust(10)
1370
- $view = $job.viewshort.ljust(10)
1371
- $iden = $job.idenshort.ljust(10)
1372
- $enabled = $job.enabled.ljust(3)
1373
- $isrecur = $job.isrecur.ljust(4)
1374
- $isrunning = $job.isrunning.ljust(4)
1375
- $iserr = $job.iserr.ljust(4)
1376
- $startcount = $lib.str.format("{startcount}", startcount=$job.startcount).ljust(7)
1377
- $laststart = $job.laststart.ljust(16)
1378
- $lastend = $job.lastend.ljust(16)
1379
-
1380
- $lib.print("{user} {iden} {view} {enabled} {isrecur} {isrunning} {iserr} {startcount} {laststart} {lastend} {query}",
1381
- user=$user, iden=$iden, view=$view, enabled=$enabled, isrecur=$isrecur,
1382
- isrunning=$isrunning, iserr=$iserr, startcount=$startcount,
1383
- laststart=$laststart, lastend=$lastend, query=$job.query)
1390
+ $row = (
1391
+ $job.user, $job.idenshort, $job.viewshort, $job.enabled,
1392
+ $job.isrecur, $job.isrunning, $job.iserr, `{$job.startcount}`,
1393
+ $job.laststart, $job.lastend, $job.query
1394
+ )
1395
+ $lib.print($printer.row($row))
1384
1396
  }
1385
1397
  } else {
1386
1398
  $lib.print("No cron jobs found")
@@ -5268,6 +5280,13 @@ class BackgroundCmd(Cmd):
5268
5280
  async for item in genr:
5269
5281
  yield item
5270
5282
 
5283
+ _query = await s_stormtypes.tostr(self.opts.query)
5284
+ query = await runt.getStormQuery(_query)
5285
+
5286
+ # make sure the subquery *could* have run
5287
+ async with runt.getSubRuntime(query) as subr:
5288
+ query.validate(subr)
5289
+
5271
5290
  runtprims = await s_stormtypes.toprim(self.runt.getScopeVars(), use_list=True)
5272
5291
  runtvars = {k: v for (k, v) in runtprims.items() if s_msgpack.isok(v)}
5273
5292
 
@@ -5277,12 +5296,6 @@ class BackgroundCmd(Cmd):
5277
5296
  'vars': runtvars,
5278
5297
  }
5279
5298
 
5280
- _query = await s_stormtypes.tostr(self.opts.query)
5281
- query = await runt.getStormQuery(_query)
5282
-
5283
- # make sure the subquery *could* have run with existing vars
5284
- query.validate(runt)
5285
-
5286
5299
  coro = self.execStormTask(query, opts)
5287
5300
  runt.snap.core.schedCoro(coro)
5288
5301
 
@@ -5339,9 +5352,12 @@ class ParallelCmd(Cmd):
5339
5352
  raise s_exc.StormRuntimeError(mesg=mesg)
5340
5353
 
5341
5354
  size = await s_stormtypes.toint(self.opts.size)
5342
- query = await runt.getStormQuery(self.opts.query)
5343
5355
 
5344
- query.validate(runt)
5356
+ _query = await s_stormtypes.tostr(self.opts.query)
5357
+ query = await runt.getStormQuery(_query)
5358
+
5359
+ async with runt.getSubRuntime(query) as subr:
5360
+ query.validate(subr)
5345
5361
 
5346
5362
  async with await s_base.Base.anit() as base:
5347
5363
 
@@ -105,6 +105,21 @@ class LibModelExt(s_stormtypes.Lib):
105
105
  'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
106
106
  ),
107
107
  'returns': {'type': 'null'}}},
108
+ {'name': 'addType', 'desc': 'Add an extended type definition to the data model.',
109
+ 'type': {'type': 'function', '_funcname': 'addType',
110
+ 'args': (
111
+ {'name': 'typename', 'type': 'str', 'desc': 'The name of the type to add.'},
112
+ {'name': 'basetype', 'type': 'str', 'desc': 'The base type the type is derived from.'},
113
+ {'name': 'typeopts', 'type': 'dict', 'desc': 'A Synapse type opts dictionary.'},
114
+ {'name': 'typeinfo', 'type': 'dict', 'desc': 'A Synapse type info dictionary.'},
115
+ ),
116
+ 'returns': {'type': 'null'}}},
117
+ {'name': 'delType', 'desc': 'Remove an extended type definition from the model.',
118
+ 'type': {'type': 'function', '_funcname': 'delType',
119
+ 'args': (
120
+ {'name': 'typename', 'type': 'str', 'desc': 'The extended type to remove.'},
121
+ ),
122
+ 'returns': {'type': 'null'}}},
108
123
  )
109
124
  _storm_lib_path = ('model', 'ext')
110
125
 
@@ -122,6 +137,8 @@ class LibModelExt(s_stormtypes.Lib):
122
137
  'addExtModel': self.addExtModel,
123
138
  'addEdge': self.addEdge,
124
139
  'delEdge': self.delEdge,
140
+ 'addType': self.addType,
141
+ 'delType': self.delType,
125
142
  }
126
143
 
127
144
  # TODO type docs in the new convention
@@ -210,6 +227,7 @@ class LibModelExt(s_stormtypes.Lib):
210
227
  return await self.runt.snap.core.getExtModel()
211
228
 
212
229
  async def addExtModel(self, model):
230
+ self.runt.reqAdmin()
213
231
  model = await s_stormtypes.toprim(model)
214
232
  return await self.runt.snap.core.addExtModel(model)
215
233
 
@@ -249,3 +267,16 @@ class LibModelExt(s_stormtypes.Lib):
249
267
 
250
268
  s_stormtypes.confirm(('model', 'edge', 'del'))
251
269
  await self.runt.snap.core.delEdge((n1form, verb, n2form))
270
+
271
+ async def addType(self, typename, basetype, typeopts, typeinfo):
272
+ typename = await s_stormtypes.tostr(typename)
273
+ basetype = await s_stormtypes.tostr(basetype)
274
+ typeopts = await s_stormtypes.toprim(typeopts)
275
+ typeinfo = await s_stormtypes.toprim(typeinfo)
276
+ s_stormtypes.confirm(('model', 'type', 'add', typename))
277
+ await self.runt.snap.core.addType(typename, basetype, typeopts, typeinfo)
278
+
279
+ async def delType(self, typename):
280
+ typename = await s_stormtypes.tostr(typename)
281
+ s_stormtypes.confirm(('model', 'type', 'del', typename))
282
+ await self.runt.snap.core.delType(typename)
@@ -155,10 +155,7 @@ class LibScrape(s_stormtypes.Lib):
155
155
  if fangs:
156
156
  _fangs = {src: dst for (src, dst) in fangs}
157
157
  _fangre = s_scrape.genFangRegex(_fangs)
158
- if len(text) < s_scrape.SCRAPE_SPAWN_LENGTH:
159
- scrape_text, offsets = s_scrape.refang_text2(text, re=_fangre, fangs=_fangs)
160
- else:
161
- scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
158
+ scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
162
159
 
163
160
  async for info in s_scrape.genMatchesAsync(scrape_text, regx, opts=opts):
164
161
  valu = info.pop('valu')
synapse/lib/stormtypes.py CHANGED
@@ -7545,6 +7545,9 @@ class View(Prim):
7545
7545
  {'name': 'parent', 'desc': 'The parent View. Will be ``$lib.null`` if the view is not a fork.', 'type': 'str'},
7546
7546
  {'name': 'triggers', 'desc': 'The ``trigger`` objects associated with the ``view``.',
7547
7547
  'type': 'list', },
7548
+ {'name': 'children', 'desc': 'Yield Views which are children of this View.',
7549
+ 'type': {'type': 'function', '_funcname': '_methGetChildren',
7550
+ 'returns': {'name': 'yields', 'type': 'view', 'desc': 'Child Views.', }}},
7548
7551
  {'name': 'set', 'desc': '''
7549
7552
  Set a view configuration option.
7550
7553
 
@@ -7832,6 +7835,7 @@ class View(Prim):
7832
7835
  'merge': self._methViewMerge,
7833
7836
  'detach': self.detach,
7834
7837
  'addNode': self.addNode,
7838
+ 'children': self._methGetChildren,
7835
7839
  'getEdges': self._methGetEdges,
7836
7840
  'wipeLayer': self._methWipeLayer,
7837
7841
  'swapLayer': self._methSwapLayer,
@@ -7967,6 +7971,12 @@ class View(Prim):
7967
7971
  async for valu in view.iterPropValues(propname):
7968
7972
  yield valu
7969
7973
 
7974
+ @stormfunc(readonly=True)
7975
+ async def _methGetChildren(self):
7976
+ view = self._reqView()
7977
+ async for child in view.children():
7978
+ yield View(self.runt, await child.pack(), path=self.path)
7979
+
7970
7980
  @stormfunc(readonly=True)
7971
7981
  async def _methGetEdges(self, verb=None):
7972
7982
  verb = await toprim(verb)
@@ -9610,7 +9620,7 @@ def fromprim(valu, path=None, basetypes=True):
9610
9620
 
9611
9621
  return valu
9612
9622
 
9613
- async def tostor(valu):
9623
+ async def tostor(valu, isndef=False):
9614
9624
 
9615
9625
  if isinstance(valu, Number):
9616
9626
  return str(valu.value())
@@ -9633,6 +9643,9 @@ async def tostor(valu):
9633
9643
  pass
9634
9644
  return retn
9635
9645
 
9646
+ if isndef and isinstance(valu, s_node.Node):
9647
+ return valu.ndef
9648
+
9636
9649
  return await toprim(valu)
9637
9650
 
9638
9651
  async def tocmprvalu(valu):
@@ -9676,6 +9689,9 @@ async def tostr(valu, noneok=False):
9676
9689
  if isinstance(valu, bytes):
9677
9690
  return valu.decode('utf8', 'surrogatepass')
9678
9691
 
9692
+ if isinstance(valu, s_node.Node):
9693
+ return valu.repr()
9694
+
9679
9695
  return str(valu)
9680
9696
  except Exception as e:
9681
9697
  mesg = f'Failed to make a string from {valu!r}.'
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 187, 0)
226
+ version = (2, 188, 1)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = '5cdabb0eeea74eff019d8bcdc461b48bb5eb6651'
228
+ commit = '2cf97aa2a3266442393dfc78525749d3f65d9fc7'
synapse/lib/view.py CHANGED
@@ -1315,6 +1315,14 @@ class View(s_nexus.Pusher): # type: ignore
1315
1315
 
1316
1316
  todo.append(child)
1317
1317
 
1318
+ async def children(self):
1319
+ for view in list(self.core.views.values()):
1320
+ if view.parent != self:
1321
+ await asyncio.sleep(0)
1322
+ continue
1323
+
1324
+ yield view
1325
+
1318
1326
  async def insertParentFork(self, useriden, name=None):
1319
1327
  '''
1320
1328
  Insert a new View between a forked View and its parent.
@@ -1614,9 +1622,7 @@ class View(s_nexus.Pusher): # type: ignore
1614
1622
  if trig is not None:
1615
1623
  return self.triggers.get(tdef['iden']).pack()
1616
1624
 
1617
- gate = self.core.auth.getAuthGate(tdef['iden'])
1618
- if gate is not None:
1619
- raise s_exc.DupIden(mesg='An AuthGate with this iden already exists')
1625
+ self.core.auth.reqNoAuthGate(tdef['iden'])
1620
1626
 
1621
1627
  user = self.core.auth.user(tdef['user'])
1622
1628
  await self.core.getStormQuery(tdef['storm'])
synapse/models/base.py CHANGED
@@ -107,6 +107,19 @@ class BaseModule(s_module.CoreModule):
107
107
 
108
108
  ('meta:sophistication', ('int', {'enums': sophenums}), {
109
109
  'doc': 'A sophistication score with named values: very low, low, medium, high, and very high.'}),
110
+
111
+ ('meta:aggregate:type:taxonomy', ('taxonomy', {}), {
112
+ 'interfaces': ('meta:taxonomy',),
113
+ 'doc': 'A type of item being counted in aggregate.'}),
114
+
115
+ ('meta:aggregate', ('guid', {}), {
116
+ 'display': {
117
+ 'columns': (
118
+ {'type': 'prop', 'opts': {'name': 'type'}},
119
+ {'type': 'prop', 'opts': {'name': 'count'}},
120
+ ),
121
+ },
122
+ 'doc': 'A node which represents an aggregate count of a specific type.'}),
110
123
  ),
111
124
  'interfaces': (
112
125
  ('meta:taxonomy', {
@@ -285,6 +298,20 @@ class BaseModule(s_module.CoreModule):
285
298
  'doc': 'An external identifier for the rule.'}),
286
299
  )),
287
300
 
301
+ ('meta:aggregate:type:taxonomy', {}, ()),
302
+ ('meta:aggregate', {}, (
303
+
304
+ ('type', ('meta:aggregate:type:taxonomy', {}), {
305
+ 'ex': 'casualties.civilian',
306
+ 'doc': 'The type of items being counted in aggregate.'}),
307
+
308
+ ('time', ('time', {}), {
309
+ 'doc': 'The time that the count was computed.'}),
310
+
311
+ ('count', ('int', {}), {
312
+ 'doc': 'The number of items counted in aggregate.'}),
313
+ )),
314
+
288
315
  ('graph:cluster', {}, (
289
316
  ('name', ('str', {'lower': True}), {
290
317
  'doc': 'A human friendly name for the cluster.'}),
synapse/models/files.py CHANGED
@@ -284,6 +284,16 @@ class FileModule(s_module.CoreModule):
284
284
  'doc': 'A parent file that fully contains the specified child file.',
285
285
  }),
286
286
 
287
+ ('file:attachment', ('guid', {}), {
288
+ 'display': {
289
+ 'columns': (
290
+ {'type': 'prop', 'opts': {'name': 'name'}},
291
+ {'type': 'prop', 'opts': {'name': 'file'}},
292
+ {'type': 'prop', 'opts': {'name': 'text'}},
293
+ ),
294
+ },
295
+ 'doc': 'A file attachment.'}),
296
+
287
297
  ('file:archive:entry', ('guid', {}), {
288
298
  'doc': 'An archive entry representing a file and metadata within a parent archive file.'}),
289
299
 
@@ -601,6 +611,18 @@ class FileModule(s_module.CoreModule):
601
611
  }),
602
612
  )),
603
613
 
614
+ ('file:attachment', {}, (
615
+
616
+ ('name', ('file:path', {}), {
617
+ 'doc': 'The name of the attached file.'}),
618
+
619
+ ('text', ('str', {}), {
620
+ 'doc': 'Any text associated with the file such as alt-text for images.'}),
621
+
622
+ ('file', ('file:bytes', {}), {
623
+ 'doc': 'The file which was attached.'}),
624
+ )),
625
+
604
626
  ('file:archive:entry', {}, (
605
627
 
606
628
  ('parent', ('file:bytes', {}), {
synapse/models/inet.py CHANGED
@@ -1369,6 +1369,9 @@ class InetModule(s_module.CoreModule):
1369
1369
  'doc': 'A username string.'
1370
1370
  }),
1371
1371
 
1372
+ ('inet:service:object', ('ndef', {'interfaces': ('inet:service:object',)}), {
1373
+ 'doc': 'An ndef type including all forms which implement the inet:service:object interface.'}),
1374
+
1372
1375
  ('inet:search:query', ('guid', {}), {
1373
1376
  'interfaces': ('inet:service:action',),
1374
1377
  'doc': 'An instance of a search query issued to a search engine.',
@@ -1530,6 +1533,14 @@ class InetModule(s_module.CoreModule):
1530
1533
  'interfaces': ('inet:service:object',),
1531
1534
  'doc': 'An account within a service platform. Accounts may be instance specific.'}),
1532
1535
 
1536
+ ('inet:service:relationship:type:taxonomy', ('taxonomy', {}), {
1537
+ 'interfaces': ('meta:taxonomy',),
1538
+ 'doc': 'A service object relationship type taxonomy.'}),
1539
+
1540
+ ('inet:service:relationship', ('guid', {}), {
1541
+ 'interfaces': ('inet:service:object',),
1542
+ 'doc': 'A relationship between two service objects.'}),
1543
+
1533
1544
  ('inet:service:permission:type:taxonomy', ('taxonomy', {}), {
1534
1545
  'interfaces': ('meta:taxonomy',),
1535
1546
  'doc': 'A permission type taxonomy.'}),
@@ -1588,6 +1599,10 @@ class InetModule(s_module.CoreModule):
1588
1599
  'interfaces': ('meta:taxonomy',),
1589
1600
  'doc': 'A message type taxonomy.'}),
1590
1601
 
1602
+ ('inet:service:emote', ('guid', {}), {
1603
+ 'interfaces': ('inet:service:object',),
1604
+ 'doc': 'An emote or reaction by an account.'}),
1605
+
1591
1606
  ('inet:service:access', ('guid', {}), {
1592
1607
  'interfaces': ('inet:service:action',),
1593
1608
  'doc': 'Represents a user access request to a service resource.'}),
@@ -2015,8 +2030,11 @@ class InetModule(s_module.CoreModule):
2015
2030
  'doc': 'The guid of the destination process.'
2016
2031
  }),
2017
2032
  ('dst:exe', ('file:bytes', {}), {
2018
- 'doc': 'The file (executable) that received the connection.'
2019
- }),
2033
+ 'doc': 'The file (executable) that received the connection.'}),
2034
+
2035
+ ('dst:txfiles', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
2036
+ 'doc': 'An array of files sent by the destination host.'}),
2037
+
2020
2038
  ('dst:txcount', ('int', {}), {
2021
2039
  'doc': 'The number of packets sent by the destination host.'
2022
2040
  }),
@@ -2049,8 +2067,11 @@ class InetModule(s_module.CoreModule):
2049
2067
  'doc': 'The guid of the source process.'
2050
2068
  }),
2051
2069
  ('src:exe', ('file:bytes', {}), {
2052
- 'doc': 'The file (executable) that created the connection.'
2053
- }),
2070
+ 'doc': 'The file (executable) that created the connection.'}),
2071
+
2072
+ ('src:txfiles', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
2073
+ 'doc': 'An array of files sent by the source host.'}),
2074
+
2054
2075
  ('src:txcount', ('int', {}), {
2055
2076
  'doc': 'The number of packets sent by the source host.'
2056
2077
  }),
@@ -3586,6 +3607,20 @@ class InetModule(s_module.CoreModule):
3586
3607
  'doc': 'Current profile details associated with the account.'}),
3587
3608
  )),
3588
3609
 
3610
+ ('inet:service:relationship:type:taxonomy', {}, ()),
3611
+ ('inet:service:relationship', {}, (
3612
+
3613
+ ('source', ('inet:service:object', {}), {
3614
+ 'doc': 'The source object.'}),
3615
+
3616
+ ('target', ('inet:service:object', {}), {
3617
+ 'doc': 'The target object.'}),
3618
+
3619
+ ('type', ('inet:service:relationship:type:taxonomy', {}), {
3620
+ 'ex': 'follows',
3621
+ 'doc': 'The type of relationship between the source and the target.'}),
3622
+ )),
3623
+
3589
3624
  ('inet:service:group', {}, ( # inet:service:object
3590
3625
 
3591
3626
  ('id', ('str', {'strip': True}), {
@@ -3743,6 +3778,16 @@ class InetModule(s_module.CoreModule):
3743
3778
  'doc': 'The file which was attached to the message.'}),
3744
3779
  )),
3745
3780
 
3781
+ ('inet:service:emote', {}, (
3782
+
3783
+ ('about', ('inet:service:object', {}), {
3784
+ 'doc': 'The node that the emote is about.'}),
3785
+
3786
+ ('text', ('str', {'strip': True}), {
3787
+ 'ex': ':partyparrot:',
3788
+ 'doc': 'The unicode or emote text of the reaction.'}),
3789
+ )),
3790
+
3746
3791
  ('inet:service:channel', {}, (
3747
3792
 
3748
3793
  ('name', ('str', {'onespace': True, 'lower': True}), {