synapse 2.186.0__py311-none-any.whl → 2.188.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (58) hide show
  1. synapse/cortex.py +133 -9
  2. synapse/datamodel.py +20 -4
  3. synapse/exc.py +14 -1
  4. synapse/lib/ast.py +6 -4
  5. synapse/lib/auth.py +9 -0
  6. synapse/lib/hive.py +1 -1
  7. synapse/lib/httpapi.py +2 -1
  8. synapse/lib/modelrev.py +771 -11
  9. synapse/lib/nexus.py +6 -0
  10. synapse/lib/node.py +5 -3
  11. synapse/lib/scrape.py +18 -104
  12. synapse/lib/spooled.py +26 -3
  13. synapse/lib/storm.py +51 -28
  14. synapse/lib/stormlib/model.py +320 -250
  15. synapse/lib/stormlib/modelext.py +31 -0
  16. synapse/lib/stormlib/scrape.py +1 -4
  17. synapse/lib/stormtypes.py +53 -11
  18. synapse/lib/version.py +2 -2
  19. synapse/lib/view.py +9 -3
  20. synapse/models/base.py +27 -0
  21. synapse/models/files.py +22 -0
  22. synapse/models/inet.py +49 -4
  23. synapse/models/infotech.py +49 -22
  24. synapse/models/orgs.py +64 -2
  25. synapse/models/proj.py +1 -6
  26. synapse/models/risk.py +65 -0
  27. synapse/tests/test_cortex.py +21 -0
  28. synapse/tests/test_lib_agenda.py +13 -0
  29. synapse/tests/test_lib_auth.py +15 -0
  30. synapse/tests/test_lib_cell.py +2 -1
  31. synapse/tests/test_lib_httpapi.py +6 -0
  32. synapse/tests/test_lib_modelrev.py +918 -379
  33. synapse/tests/test_lib_nexus.py +26 -0
  34. synapse/tests/test_lib_scrape.py +14 -6
  35. synapse/tests/test_lib_spooled.py +34 -0
  36. synapse/tests/test_lib_storm.py +48 -0
  37. synapse/tests/test_lib_stormlib_model.py +0 -270
  38. synapse/tests/test_lib_stormlib_modelext.py +76 -1
  39. synapse/tests/test_lib_stormlib_scrape.py +0 -8
  40. synapse/tests/test_lib_stormtypes.py +12 -1
  41. synapse/tests/test_lib_trigger.py +8 -0
  42. synapse/tests/test_lib_view.py +24 -0
  43. synapse/tests/test_model_base.py +11 -0
  44. synapse/tests/test_model_files.py +19 -0
  45. synapse/tests/test_model_inet.py +33 -0
  46. synapse/tests/test_model_infotech.py +14 -11
  47. synapse/tests/test_model_orgs.py +39 -0
  48. synapse/tests/test_model_proj.py +11 -1
  49. synapse/tests/test_model_risk.py +32 -0
  50. synapse/tools/changelog.py +11 -3
  51. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/METADATA +1 -1
  52. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/RECORD +55 -58
  53. synapse/assets/__init__.py +0 -35
  54. synapse/assets/storm/migrations/model-0.2.28.storm +0 -355
  55. synapse/tests/test_assets.py +0 -25
  56. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/LICENSE +0 -0
  57. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/WHEEL +0 -0
  58. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/top_level.txt +0 -0
@@ -264,6 +264,14 @@ class TrigTest(s_t_utils.SynTest):
264
264
  await view.addTrigger(tdef)
265
265
  self.eq(pdef0.get('storm'), (await view.getTrigger(iden)).tdef.get('storm'))
266
266
 
267
+ with self.raises(s_exc.DupIden):
268
+ tdef = {'cond': 'node:add', 'storm': '[ +#dupiden ]', 'form': 'test:int', 'iden': view.iden}
269
+ await view.addTrigger(tdef)
270
+
271
+ with self.raises(s_exc.NoSuchIden):
272
+ await view.delTrigger(view.iden)
273
+ self.nn(core.auth.getAuthGate(view.iden))
274
+
267
275
  # Bad trigger parms
268
276
  with self.raises(s_exc.SchemaViolation):
269
277
  await view.addTrigger({'cond': 'nocond', 'storm': 'test:int=4', 'form': 'test:str'})
@@ -905,3 +905,27 @@ class ViewTest(s_t_utils.SynTest):
905
905
 
906
906
  with self.raises(s_exc.BadState):
907
907
  await core.callStorm('return($lib.view.get().insertParentFork().iden)')
908
+
909
+ async def test_view_children(self):
910
+
911
+ async with self.getTestCore() as core:
912
+
913
+ view00 = core.getView()
914
+ view01 = core.getView((await view00.fork())['iden'])
915
+ view02 = core.getView((await view01.fork())['iden'])
916
+ view03 = core.getView((await view01.fork())['iden'])
917
+
918
+ q = '''
919
+ $kids = ([])
920
+ for $child in $lib.view.get($iden).children() { $kids.append($child.iden) }
921
+ return($kids)
922
+ '''
923
+
924
+ opts = {'vars': {'iden': view00.iden}}
925
+ self.eq([view01.iden], await core.callStorm(q, opts=opts))
926
+
927
+ opts['vars']['iden'] = view01.iden
928
+ self.eq([view02.iden, view03.iden], await core.callStorm(q, opts=opts))
929
+
930
+ opts['vars']['iden'] = view02.iden
931
+ self.eq([], await core.callStorm(q, opts=opts))
@@ -391,3 +391,14 @@ class BaseTest(s_t_utils.SynTest):
391
391
  for node in nodes:
392
392
  form = core.model.form(node.ndef[1])
393
393
  self.true(form.deprecated, msg=form)
394
+
395
+ async def test_model_aggregate(self):
396
+
397
+ async with self.getTestCore() as core:
398
+
399
+ nodes = await core.nodes('[ meta:aggregate=* :count=99 :type=bottles :time=20240202 ]')
400
+ self.len(1, nodes)
401
+ self.eq(99, nodes[0].get('count'))
402
+ self.eq('bottles.', nodes[0].get('type'))
403
+ self.eq(1706832000000, nodes[0].get('time'))
404
+ self.len(1, await core.nodes('meta:aggregate -> meta:aggregate:type:taxonomy'))
@@ -638,3 +638,22 @@ class FileTest(s_t_utils.SynTest):
638
638
  self.eq(node.get('iconindex'), 1)
639
639
 
640
640
  self.len(1, await core.nodes('file:mime:lnk -> it:hostname'))
641
+
642
+ async def test_model_file_attachment(self):
643
+
644
+ async with self.getTestCore() as core:
645
+
646
+ nodes = await core.nodes('''
647
+ [ file:attachment=*
648
+ :name=Foo/Bar.exe
649
+ :text="foo bar"
650
+ :file=*
651
+ ]
652
+ ''')
653
+ self.len(1, nodes)
654
+ self.nn(nodes[0].get('file'))
655
+ self.eq('foo bar', nodes[0].get('text'))
656
+ self.eq('foo/bar.exe', nodes[0].get('name'))
657
+
658
+ self.len(1, await core.nodes('file:attachment -> file:bytes'))
659
+ self.len(1, await core.nodes('file:attachment -> file:path'))
@@ -455,6 +455,8 @@ class InetModelTest(s_t_utils.SynTest):
455
455
  :src:rdp:hostname=SYNCODER
456
456
  :src:rdp:keyboard:layout=AZERTY
457
457
  :raw=((10), (20))
458
+ :src:txfiles={[ file:attachment=* :name=foo.exe ]}
459
+ :dst:txfiles={[ file:attachment=* :name=bar.exe ]}
458
460
  )]'''
459
461
  nodes = await core.nodes(q, opts={'vars': {'valu': valu, 'p': props}})
460
462
  self.len(1, nodes)
@@ -501,6 +503,8 @@ class InetModelTest(s_t_utils.SynTest):
501
503
  self.len(2, await core.nodes('inet:flow -> crypto:x509:cert'))
502
504
  self.len(1, await core.nodes('inet:flow :src:ssh:key -> crypto:key'))
503
505
  self.len(1, await core.nodes('inet:flow :dst:ssh:key -> crypto:key'))
506
+ self.len(1, await core.nodes('inet:flow :src:txfiles -> file:attachment +:name=foo.exe'))
507
+ self.len(1, await core.nodes('inet:flow :dst:txfiles -> file:attachment +:name=bar.exe'))
504
508
 
505
509
  async def test_fqdn(self):
506
510
  formname = 'inet:fqdn'
@@ -3374,3 +3378,32 @@ class InetModelTest(s_t_utils.SynTest):
3374
3378
  :channel -> inet:service:channel
3375
3379
  +:name="/r/synapse"
3376
3380
  '''))
3381
+
3382
+ nodes = await core.nodes('''
3383
+ [ inet:service:relationship=*
3384
+ :source={ inet:service:account:user=visi }
3385
+ :target={ inet:service:account:user=visi }
3386
+ :type=follows
3387
+ ]
3388
+ ''')
3389
+ self.nn(nodes[0].get('source'))
3390
+ self.nn(nodes[0].get('target'))
3391
+ self.eq('follows.', nodes[0].get('type'))
3392
+ self.len(1, await core.nodes('inet:service:relationship :source -> inet:service:account +:user=visi'))
3393
+ self.len(1, await core.nodes('inet:service:relationship :target -> inet:service:account +:user=visi'))
3394
+
3395
+ nodes = await core.nodes('''
3396
+ [ inet:service:emote=*
3397
+ :creator={ inet:service:account:user=visi }
3398
+ :about={[ it:dev:repo=* :name=vertex ]}
3399
+ :text=":gothparrot:"
3400
+ ]
3401
+ ''')
3402
+ self.nn(nodes[0].get('about'))
3403
+ self.nn(nodes[0].get('creator'))
3404
+ self.eq(':gothparrot:', nodes[0].get('text'))
3405
+ self.len(1, await core.nodes('inet:service:emote :about -> it:dev:repo +:name=vertex'))
3406
+ self.len(1, await core.nodes('inet:service:emote :creator -> inet:service:account +:user=visi'))
3407
+
3408
+ with self.raises(s_exc.BadTypeValu):
3409
+ await core.nodes('[ inet:service:relationship=* :source={[it:dev:str=foo]} ]')
@@ -1778,8 +1778,17 @@ class InfotechModelTest(s_t_utils.SynTest):
1778
1778
  with self.raises(s_exc.BadTypeValu):
1779
1779
  nodes = await core.nodes('[it:sec:cpe=cpe:2.3:1:2:3:4:5:6:7:8:9:10:11:12]')
1780
1780
 
1781
- nodes = await core.nodes('[ it:sec:cpe=cpe:2.3:a:vertex:synapse ]')
1782
- self.eq(nodes[0].ndef, ('it:sec:cpe', 'cpe:2.3:a:vertex:synapse:*:*:*:*:*:*:*:*'))
1781
+ with self.raises(s_exc.BadTypeValu):
1782
+ await core.nodes('[ it:sec:cpe=cpe:2.3:a:vertex:synapse ]')
1783
+
1784
+ with self.raises(s_exc.BadTypeValu):
1785
+ await core.callStorm(r'$lib.cast(it:sec:cpe, "cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*")')
1786
+
1787
+ with self.raises(s_exc.BadTypeValu):
1788
+ await core.callStorm(r'$lib.cast(it:sec:cpe:v2_2, "cpe:/a:01generator:pireospay\r\n:-::~~~prestashop~~")')
1789
+
1790
+ with self.raises(s_exc.BadTypeValu):
1791
+ await core.callStorm('$lib.cast(it:sec:cpe:v2_2, "cpe:2.3:*")')
1783
1792
 
1784
1793
  nodes = await core.nodes('''[
1785
1794
  it:sec:cpe=cpe:2.3:a:microsoft:internet_explorer:8.0.6001:beta:*:*:*:*:*:*
@@ -1792,14 +1801,8 @@ class InfotechModelTest(s_t_utils.SynTest):
1792
1801
  self.eq(nodes[0].get('version'), '8.0.6001')
1793
1802
  self.eq(nodes[0].get('update'), 'beta')
1794
1803
 
1795
- nodes = await core.nodes("[ it:sec:cpe='cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*' ]")
1796
- self.len(1, nodes)
1797
- self.eq(nodes[0].ndef, ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:7.4:*:*:*:*:*:*:*'))
1798
- self.eq(nodes[0].get('part'), 'a')
1799
- self.eq(nodes[0].get('product'), 'openssh')
1800
- self.eq(nodes[0].get('vendor'), 'openbsd')
1801
- self.eq(nodes[0].get('version'), '7.4')
1802
- self.eq(nodes[0].get('v2_2'), 'cpe:/a:openbsd:openssh:7.4')
1804
+ with self.raises(s_exc.BadTypeValu):
1805
+ await core.nodes("[ it:sec:cpe='cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*' ]")
1803
1806
 
1804
1807
  nodes = await core.nodes(r'[ it:sec:cpe="cpe:2.3:o:cisco:ios:12.1\(22\)ea1a:*:*:*:*:*:*:*" ]')
1805
1808
  self.len(1, nodes)
@@ -1923,7 +1926,7 @@ class InfotechModelTest(s_t_utils.SynTest):
1923
1926
  nodes = await core.nodes(q, opts={'vars': {'valu': valu}})
1924
1927
  self.len(1, nodes)
1925
1928
  node = nodes[0]
1926
- self.eq(node.ndef[1], valu.lower())
1929
+ self.eq(node.ndef[1], valu.lower(), msg=valu.lower())
1927
1930
 
1928
1931
  async def test_infotech_c2config(self):
1929
1932
  async with self.getTestCore() as core:
@@ -742,6 +742,32 @@ class OuModelTest(s_t_utils.SynTest):
742
742
  self.len(1, await core.nodes('ou:enacted :ext:creator -> ps:contact +:name=root'))
743
743
  self.len(1, await core.nodes('ou:enacted :ext:assignee -> ps:contact +:name=visi'))
744
744
 
745
+ nodes = await core.nodes('''
746
+ [ ou:candidate=*
747
+ :org={ ou:org:name=vertex | limit 1 }
748
+ :contact={ ps:contact:name=visi | limit 1 }
749
+ :intro=" Hi there!"
750
+ :submitted=20241104
751
+ :method=referral.employee
752
+ :resume=*
753
+ :opening=*
754
+ :agent={[ ps:contact=* :name=agent ]}
755
+ :recruiter={[ ps:contact=* :name=recruiter ]}
756
+ :attachments={[ file:attachment=* :name=questions.pdf ]}
757
+ ]
758
+ ''')
759
+ self.len(1, nodes)
760
+ self.eq('Hi there!', nodes[0].get('intro'))
761
+ self.eq(1730678400000, nodes[0].get('submitted'))
762
+ self.eq('referral.employee.', nodes[0].get('method'))
763
+ self.len(1, await core.nodes('ou:candidate :org -> ou:org +:name=vertex'))
764
+ self.len(1, await core.nodes('ou:candidate :agent -> ps:contact +:name=agent'))
765
+ self.len(1, await core.nodes('ou:candidate :contact -> ps:contact +:name=visi'))
766
+ self.len(1, await core.nodes('ou:candidate :recruiter -> ps:contact +:name=recruiter'))
767
+
768
+ self.len(1, await core.nodes('ou:candidate :method -> ou:candidate:method:taxonomy'))
769
+ self.len(1, await core.nodes('ou:candidate :attachments -> file:attachment'))
770
+
745
771
  async def test_ou_code_prefixes(self):
746
772
  guid0 = s_common.guid()
747
773
  guid1 = s_common.guid()
@@ -839,16 +865,21 @@ class OuModelTest(s_t_utils.SynTest):
839
865
  :sic="1234,5678"
840
866
  :isic=C1393
841
867
  :desc="Moldy cheese"
868
+ :reporter={[ ou:org=* :name=vertex ]}
869
+ :reporter:name=vertex
842
870
  ] '''
843
871
  nodes = await core.nodes(q)
844
872
  self.len(1, nodes)
873
+ self.nn(nodes[0].get('reporter'))
845
874
  self.eq('foo bar', nodes[0].get('name'))
875
+ self.eq('vertex', nodes[0].get('reporter:name'))
846
876
  self.sorteq(('1234', '5678'), nodes[0].get('sic'))
847
877
  self.sorteq(('11111', '22222'), nodes[0].get('naics'))
848
878
  self.sorteq(('C1393', ), nodes[0].get('isic'))
849
879
  self.len(2, nodes[0].get('subs'))
850
880
  self.eq('Moldy cheese', nodes[0].get('desc'))
851
881
 
882
+ self.len(1, await core.nodes('ou:industry :reporter -> ou:org'))
852
883
  nodes = await core.nodes('ou:industry:name="foo bar" | tree { :subs -> ou:industry } | uniq')
853
884
  self.len(3, nodes)
854
885
  self.len(3, await core.nodes('ou:industryname=baz -> ou:industry -> ou:industryname'))
@@ -989,3 +1020,11 @@ class OuModelTest(s_t_utils.SynTest):
989
1020
  self.len(1, await core.nodes('ou:contribution -> econ:acct:payment'))
990
1021
  self.len(1, await core.nodes('ou:contribution -> mat:spec'))
991
1022
  self.len(1, await core.nodes('ou:contribution -> ou:jobtitle +ou:jobtitle=analysts'))
1023
+
1024
+ async def test_ou_technique(self):
1025
+
1026
+ async with self.getTestCore() as core:
1027
+ nodes = await core.nodes('''
1028
+ [ ou:technique=* :name=foo +(uses)> { [ risk:vuln=* :name=bar ] } ]
1029
+ ''')
1030
+ self.len(1, await core.nodes('ou:technique:name=foo -(uses)> risk:vuln:name=bar'))
@@ -334,7 +334,7 @@ class ProjModelTest(s_test.SynTest):
334
334
  self.true(await core.callStorm('return($lib.projects.del($proj))', opts=opts))
335
335
  self.false(await core.callStorm('return($lib.projects.del(newp))', opts=opts))
336
336
 
337
- self.none(core.auth.getAuthGate(proj))
337
+ self.nn(core.auth.getAuthGate(proj))
338
338
 
339
339
  self.len(1, await core.nodes('yield $lib.projects.add(proj)'))
340
340
  self.len(1, await core.nodes('yield $lib.projects.get(proj).epics.add(epic)'))
@@ -345,6 +345,16 @@ class ProjModelTest(s_test.SynTest):
345
345
  name = await core.callStorm('$p=$lib.projects.get(proj) $p.name=newproj return ( $p.name )')
346
346
  self.eq(name, 'newproj')
347
347
 
348
+ viewiden = core.getView().iden
349
+ self.len(1, await core.nodes(f'[ proj:project={viewiden}]'))
350
+ gate = core.auth.getAuthGate(viewiden)
351
+ self.eq(gate.type, 'view')
352
+
353
+ await core.nodes(f'proj:project={viewiden} | delnode')
354
+ gate = core.auth.getAuthGate(viewiden)
355
+ self.nn(gate)
356
+ self.eq(gate.type, 'view')
357
+
348
358
  async def test_model_proj_attachment(self):
349
359
 
350
360
  async with self.getTestCore() as core:
@@ -536,6 +536,31 @@ class RiskModelTest(s_t_utils.SynTest):
536
536
  self.len(1, await core.nodes('risk:vuln:name=redtree -> risk:vulnerable :node -> *'))
537
537
  self.len(1, await core.nodes('risk:vulnerable -> risk:mitigation'))
538
538
 
539
+ nodes = await core.nodes('''
540
+ [ risk:outage=*
541
+ :name="The Big One"
542
+ :period=(2023, 2024)
543
+ :type=service.power
544
+ :cause=nature.earthquake
545
+ :provider={[ ou:org=* :name="desert power" ]}
546
+ :provider:name="desert power"
547
+ :reporter={ ou:org:name=vertex }
548
+ :reporter:name=vertex
549
+ ]
550
+ ''')
551
+ self.len(1, nodes)
552
+ self.nn(nodes[0].get('reporter'))
553
+ self.eq('the big one', nodes[0].get('name'))
554
+ self.eq('vertex', nodes[0].get('reporter:name'))
555
+ self.eq('desert power', nodes[0].get('provider:name'))
556
+ self.eq('service.power.', nodes[0].get('type'))
557
+ self.eq('nature.earthquake.', nodes[0].get('cause'))
558
+ self.eq((1672531200000, 1704067200000), nodes[0].get('period'))
559
+
560
+ self.len(1, await core.nodes('risk:outage -> risk:outage:cause:taxonomy'))
561
+ self.len(1, await core.nodes('risk:outage :reporter -> ou:org +:name=vertex'))
562
+ self.len(1, await core.nodes('risk:outage :provider -> ou:org +:name="desert power"'))
563
+
539
564
  async def test_model_risk_mitigation(self):
540
565
  async with self.getTestCore() as core:
541
566
  nodes = await core.nodes('''[
@@ -618,3 +643,10 @@ class RiskModelTest(s_t_utils.SynTest):
618
643
  self.nn(nodes[0].get('version:min'))
619
644
  self.nn(nodes[0].get('version:max'))
620
645
  self.len(2, await core.nodes('risk:vuln:name=woot -> risk:vuln:soft:range -> it:prod:softver'))
646
+
647
+ async def test_model_risk_vuln_technique(self):
648
+ async with self.getTestCore() as core:
649
+ nodes = await core.nodes('''
650
+ [ risk:vuln=* :name=foo <(uses)+ { [ ou:technique=* :name=bar ] } ]
651
+ ''')
652
+ self.len(1, await core.nodes('risk:vuln:name=foo <(uses)- ou:technique:name=bar'))
@@ -883,9 +883,17 @@ async def format(opts: argparse.Namespace,
883
883
  text = text + f'\n{header}\n{"-" * len(header)}'
884
884
  dataz.sort(key=lambda x: x.get('prs'))
885
885
  for data in dataz:
886
- desc = data.get('desc')
887
- for line in textwrap.wrap(desc, initial_indent='- ', subsequent_indent=' ', width=opts.width):
888
- text = f'{text}\n{line}'
886
+ desc = data.get('desc') # type: str
887
+ desc_lines = desc.splitlines()
888
+ for i, chunk in enumerate(desc_lines):
889
+ if i == 0:
890
+ for line in textwrap.wrap(chunk, initial_indent='- ', subsequent_indent=' ', width=opts.width):
891
+ text = f'{text}\n{line}'
892
+ else:
893
+ text = text + '\n'
894
+ for line in textwrap.wrap(chunk, initial_indent=' ', subsequent_indent=' ', width=opts.width):
895
+ text = f'{text}\n{line}'
896
+
889
897
  if not opts.hide_prs:
890
898
  for pr in data.get('prs'):
891
899
  text = f'{text}\n (`#{pr} <https://github.com/vertexproject/synapse/pull/{pr}>`_)'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: synapse
3
- Version: 2.186.0
3
+ Version: 2.188.0
4
4
  Summary: Synapse Intelligence Analysis Framework
5
5
  Author-email: The Vertex Project LLC <root@vertex.link>
6
6
  License: Apache License 2.0