synapse 2.154.1__py311-none-any.whl → 2.156.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (74) hide show
  1. synapse/cmds/cortex.py +2 -14
  2. synapse/common.py +13 -36
  3. synapse/cortex.py +15 -508
  4. synapse/lib/ast.py +215 -22
  5. synapse/lib/cell.py +35 -8
  6. synapse/lib/certdir.py +11 -0
  7. synapse/lib/cmdr.py +0 -5
  8. synapse/lib/gis.py +2 -2
  9. synapse/lib/httpapi.py +14 -43
  10. synapse/lib/layer.py +64 -201
  11. synapse/lib/lmdbslab.py +11 -0
  12. synapse/lib/node.py +1 -3
  13. synapse/lib/parser.py +10 -0
  14. synapse/lib/slabseqn.py +2 -1
  15. synapse/lib/snap.py +121 -21
  16. synapse/lib/spooled.py +9 -0
  17. synapse/lib/storm.lark +23 -6
  18. synapse/lib/storm.py +16 -339
  19. synapse/lib/storm_format.py +5 -0
  20. synapse/lib/stormhttp.py +10 -1
  21. synapse/lib/stormlib/gen.py +1 -2
  22. synapse/lib/stormlib/gis.py +41 -0
  23. synapse/lib/stormlib/graph.py +2 -1
  24. synapse/lib/stormlib/stats.py +21 -2
  25. synapse/lib/stormlib/storm.py +16 -1
  26. synapse/lib/stormtypes.py +244 -16
  27. synapse/lib/types.py +16 -2
  28. synapse/lib/version.py +2 -2
  29. synapse/lib/view.py +118 -25
  30. synapse/models/base.py +2 -2
  31. synapse/models/inet.py +60 -30
  32. synapse/models/infotech.py +130 -8
  33. synapse/models/orgs.py +3 -0
  34. synapse/models/proj.py +3 -0
  35. synapse/models/risk.py +24 -6
  36. synapse/models/syn.py +0 -38
  37. synapse/tests/test_cmds_cortex.py +1 -1
  38. synapse/tests/test_cortex.py +70 -338
  39. synapse/tests/test_lib_agenda.py +19 -54
  40. synapse/tests/test_lib_aha.py +97 -0
  41. synapse/tests/test_lib_ast.py +596 -0
  42. synapse/tests/test_lib_grammar.py +30 -10
  43. synapse/tests/test_lib_httpapi.py +33 -49
  44. synapse/tests/test_lib_layer.py +19 -234
  45. synapse/tests/test_lib_lmdbslab.py +22 -0
  46. synapse/tests/test_lib_snap.py +9 -0
  47. synapse/tests/test_lib_spooled.py +4 -0
  48. synapse/tests/test_lib_storm.py +16 -309
  49. synapse/tests/test_lib_stormlib_gis.py +21 -0
  50. synapse/tests/test_lib_stormlib_stats.py +107 -20
  51. synapse/tests/test_lib_stormlib_storm.py +25 -0
  52. synapse/tests/test_lib_stormtypes.py +253 -8
  53. synapse/tests/test_lib_types.py +40 -0
  54. synapse/tests/test_lib_view.py +6 -13
  55. synapse/tests/test_model_base.py +1 -1
  56. synapse/tests/test_model_inet.py +15 -0
  57. synapse/tests/test_model_infotech.py +110 -0
  58. synapse/tests/test_model_orgs.py +10 -0
  59. synapse/tests/test_model_person.py +0 -3
  60. synapse/tests/test_model_proj.py +2 -1
  61. synapse/tests/test_model_risk.py +24 -0
  62. synapse/tests/test_model_syn.py +20 -34
  63. synapse/tests/test_tools_csvtool.py +2 -1
  64. synapse/tests/test_tools_feed.py +4 -30
  65. synapse/tools/csvtool.py +2 -1
  66. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/METADATA +9 -9
  67. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/RECORD +70 -72
  68. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/WHEEL +1 -1
  69. synapse/cmds/cron.py +0 -726
  70. synapse/cmds/trigger.py +0 -319
  71. synapse/tests/test_cmds_cron.py +0 -453
  72. synapse/tests/test_cmds_trigger.py +0 -176
  73. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/LICENSE +0 -0
  74. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/top_level.txt +0 -0
@@ -955,6 +955,28 @@ class StormTypesTest(s_test.SynTest):
955
955
  msgs = await core.stormlist('$lib.print($lib.undef)')
956
956
  self.stormIsInPrint('$lib.undef', msgs)
957
957
 
958
+ msgs = await core.stormlist('auth.user.add visi')
959
+ self.stormHasNoWarnErr(msgs)
960
+ msgs = await core.stormlist('auth.user.addrule visi view.read')
961
+ self.stormHasNoWarnErr(msgs)
962
+
963
+ fork00 = await core.callStorm('return($lib.view.get().fork())')
964
+ iden = fork00.get('iden')
965
+
966
+ visi = await core.auth.getUserByName('visi')
967
+ with self.raises(s_exc.AuthDeny):
968
+ await core.nodes('$lib.view.get().detach()', opts={'user': visi.iden, 'view': iden})
969
+
970
+ await core.stormlist('$lib.view.get().detach()', opts={'view': iden})
971
+
972
+ with self.raises(s_exc.BadArg):
973
+ await core.nodes('$lib.view.get().detach()', opts={'view': iden})
974
+
975
+ view = core.reqView(iden)
976
+
977
+ self.none(view.parent)
978
+ self.none(view.info.get('parent'))
979
+
958
980
  async def test_storm_lib_ps(self):
959
981
 
960
982
  async with self.getTestCore() as core:
@@ -4042,6 +4064,15 @@ class StormTypesTest(s_test.SynTest):
4042
4064
  self.nn(iden)
4043
4065
  self.true(visi.allowed(('view', 'read'), gateiden=iden))
4044
4066
 
4067
+ await visi.addRule((True, ('view', 'add')))
4068
+
4069
+ msgs = await core.stormlist('$lib.view.get().fork()', opts={'user': visi.iden})
4070
+ self.stormHasNoWarnErr(msgs)
4071
+
4072
+ await visi.addRule((False, ('view', 'fork')), gateiden=core.view.iden)
4073
+ msgs = await core.stormlist('$lib.view.get().fork()', opts={'user': visi.iden})
4074
+ self.stormIsInErr('must have permission view.fork', msgs)
4075
+
4045
4076
  async def test_storm_view_deporder(self):
4046
4077
 
4047
4078
  async with self.getTestCore() as core:
@@ -4448,6 +4479,7 @@ class StormTypesTest(s_test.SynTest):
4448
4479
 
4449
4480
  async def test_storm_lib_cron_notime(self):
4450
4481
  # test cron APIs that don't require time stepping
4482
+
4451
4483
  async with self.getTestCore() as core:
4452
4484
 
4453
4485
  cdef = await core.callStorm('return($lib.cron.add(query="{[graph:node=*]}", hourly=30).pack())')
@@ -4477,13 +4509,11 @@ class StormTypesTest(s_test.SynTest):
4477
4509
  self.eq('mydoc', cdef.get('doc'))
4478
4510
  self.eq('myname', cdef.get('name'))
4479
4511
 
4480
- async with core.getLocalProxy() as proxy:
4481
-
4482
- cdef = await proxy.editCronJob(iden0, 'name', 'lolz')
4483
- self.eq('lolz', cdef.get('name'))
4512
+ cdef = await core.callStorm('$cron=$lib.cron.get($iden) return ( $cron.set(name, lolz) )', opts=opts)
4513
+ self.eq('lolz', cdef.get('name'))
4484
4514
 
4485
- cdef = await proxy.editCronJob(iden0, 'doc', 'zoinks')
4486
- self.eq('zoinks', cdef.get('doc'))
4515
+ cdef = await core.callStorm('$cron=$lib.cron.get($iden) return ( $cron.set(doc, zoinks) )', opts=opts)
4516
+ self.eq('zoinks', cdef.get('doc'))
4487
4517
 
4488
4518
  async def test_storm_lib_cron(self):
4489
4519
 
@@ -4570,8 +4600,6 @@ class StormTypesTest(s_test.SynTest):
4570
4600
  self.stormIsInErr("Unexpected token '}' at line 1, column 10", mesgs)
4571
4601
 
4572
4602
  ##################
4573
- oldsplicespos = (await alist(prox.splices(None, 1000)))[-1][0][0]
4574
- nextoffs = (oldsplicespos + 1, 0, 0)
4575
4603
  layr = core.getLayer()
4576
4604
  nextlayroffs = await layr.getEditOffs() + 1
4577
4605
 
@@ -5897,6 +5925,7 @@ class StormTypesTest(s_test.SynTest):
5897
5925
 
5898
5926
  async def test_stormtypes_layer_counts(self):
5899
5927
  async with self.getTestCore() as core:
5928
+
5900
5929
  self.eq(0, await core.callStorm('return($lib.layer.get().getTagCount(foo.bar))'))
5901
5930
  await core.nodes('[ inet:ipv4=1.2.3.4 inet:ipv4=5.6.7.8 :asn=20 inet:asn=20 +#foo.bar ]')
5902
5931
  self.eq(0, await core.callStorm('return($lib.layer.get().getPropCount(ps:person))'))
@@ -5915,6 +5944,111 @@ class StormTypesTest(s_test.SynTest):
5915
5944
  with self.raises(s_exc.NoSuchProp):
5916
5945
  await core.callStorm("return($lib.layer.get().getPropCount('.newp'))")
5917
5946
 
5947
+ await core.nodes('.created | delnode --force')
5948
+
5949
+ await core.addTagProp('score', ('int', {}), {})
5950
+
5951
+ q = '''[
5952
+ inet:ipv4=1
5953
+ inet:ipv4=2
5954
+ inet:ipv4=3
5955
+ :asn=4
5956
+
5957
+ (ou:org=*
5958
+ ou:org=*
5959
+ :names=(foo, bar))
5960
+
5961
+ .seen=2020
5962
+ .univarray=(1, 2)
5963
+ +#foo:score=2
5964
+
5965
+ test:arrayform=(1,2,3)
5966
+ test:arrayform=(2,3,4)
5967
+ ]'''
5968
+ await core.nodes(q)
5969
+
5970
+ q = 'return($lib.layer.get().getPropCount(inet:ipv4:asn, valu=1))'
5971
+ self.eq(0, await core.callStorm(q))
5972
+
5973
+ q = 'return($lib.layer.get().getPropCount(inet:ipv4:loc, valu=1))'
5974
+ self.eq(0, await core.callStorm(q))
5975
+
5976
+ q = 'return($lib.layer.get().getPropCount(inet:ipv4:asn, valu=4))'
5977
+ self.eq(3, await core.callStorm(q))
5978
+
5979
+ q = 'return($lib.layer.get().getPropCount(inet:ipv4.seen, valu=2020))'
5980
+ self.eq(3, await core.callStorm(q))
5981
+
5982
+ q = 'return($lib.layer.get().getPropCount(".seen", valu=2020))'
5983
+ self.eq(5, await core.callStorm(q))
5984
+
5985
+ q = 'return($lib.layer.get().getPropCount(".test:univ", valu=1))'
5986
+ self.eq(0, await core.callStorm(q))
5987
+
5988
+ q = 'return($lib.layer.get().getPropCount(inet:ipv4, valu=1))'
5989
+ self.eq(1, await core.callStorm(q))
5990
+
5991
+ q = 'return($lib.layer.get().getPropCount(ou:org:names, valu=(foo, bar)))'
5992
+ self.eq(2, await core.callStorm(q))
5993
+
5994
+ q = 'return($lib.layer.get().getPropCount(".univarray", valu=(1, 2)))'
5995
+ self.eq(5, await core.callStorm(q))
5996
+
5997
+ with self.raises(s_exc.NoSuchProp):
5998
+ q = 'return($lib.layer.get().getPropCount(newp, valu=1))'
5999
+ await core.callStorm(q)
6000
+
6001
+ q = 'return($lib.layer.get().getPropArrayCount(ou:org:names))'
6002
+ self.eq(4, await core.callStorm(q))
6003
+
6004
+ q = 'return($lib.layer.get().getPropArrayCount(ou:org:names, valu=foo))'
6005
+ self.eq(2, await core.callStorm(q))
6006
+
6007
+ q = 'return($lib.layer.get().getPropArrayCount(".univarray"))'
6008
+ self.eq(10, await core.callStorm(q))
6009
+
6010
+ q = 'return($lib.layer.get().getPropArrayCount(".univarray", valu=2))'
6011
+ self.eq(5, await core.callStorm(q))
6012
+
6013
+ q = 'return($lib.layer.get().getPropArrayCount(test:arrayform))'
6014
+ self.eq(6, await core.callStorm(q))
6015
+
6016
+ q = 'return($lib.layer.get().getPropArrayCount(test:arrayform, valu=2))'
6017
+ self.eq(2, await core.callStorm(q))
6018
+
6019
+ q = 'return($lib.layer.get().getPropArrayCount(ou:org:subs))'
6020
+ self.eq(0, await core.callStorm(q))
6021
+
6022
+ q = 'return($lib.layer.get().getPropArrayCount(ou:org:subs, valu=*))'
6023
+ self.eq(0, await core.callStorm(q))
6024
+
6025
+ with self.raises(s_exc.NoSuchProp):
6026
+ q = 'return($lib.layer.get().getPropArrayCount(newp, valu=1))'
6027
+ await core.callStorm(q)
6028
+
6029
+ with self.raises(s_exc.BadTypeValu):
6030
+ q = 'return($lib.layer.get().getPropArrayCount(inet:ipv4, valu=1))'
6031
+ await core.callStorm(q)
6032
+
6033
+ q = 'return($lib.layer.get().getTagPropCount(foo, score))'
6034
+ self.eq(5, await core.callStorm(q))
6035
+
6036
+ q = 'return($lib.layer.get().getTagPropCount(foo, score, valu=2))'
6037
+ self.eq(5, await core.callStorm(q))
6038
+
6039
+ q = 'return($lib.layer.get().getTagPropCount(foo, score, form=ou:org, valu=2))'
6040
+ self.eq(2, await core.callStorm(q))
6041
+
6042
+ q = 'return($lib.layer.get().getTagPropCount(bar, score))'
6043
+ self.eq(0, await core.callStorm(q))
6044
+
6045
+ q = 'return($lib.layer.get().getTagPropCount(bar, score, valu=2))'
6046
+ self.eq(0, await core.callStorm(q))
6047
+
6048
+ with self.raises(s_exc.NoSuchTagProp):
6049
+ q = 'return($lib.layer.get().getTagPropCount(foo, newp, valu=2))'
6050
+ await core.callStorm(q)
6051
+
5918
6052
  async def test_lib_stormtypes_cmdopts(self):
5919
6053
  pdef = {
5920
6054
  'name': 'foo',
@@ -6742,3 +6876,114 @@ words\tword\twrd'''
6742
6876
  '''
6743
6877
  msgs = await core.stormlist(q, opts={'readonly': True, 'vars': {'iden': user}})
6744
6878
  self.stormIsInErr(mesg, msgs)
6879
+
6880
+ async def test_storm_view_counts(self):
6881
+
6882
+ async with self.getTestCore() as core:
6883
+
6884
+ await core.addTagProp('score', ('int', {}), {})
6885
+
6886
+ view2 = await core.view.fork()
6887
+ forkopts = {'view': view2['iden']}
6888
+
6889
+ q = '''[
6890
+ inet:ipv4=1
6891
+ inet:ipv4=2
6892
+ inet:ipv4=3
6893
+ :asn=4
6894
+
6895
+ (ou:org=*
6896
+ ou:org=*
6897
+ :names=(foo, bar))
6898
+
6899
+ .seen=2020
6900
+ .univarray=(1, 2)
6901
+ +#foo:score=2
6902
+
6903
+ test:arrayform=(1,2,3)
6904
+ test:arrayform=(2,3,4)
6905
+ ]'''
6906
+ await core.nodes(q)
6907
+
6908
+ q = '''[
6909
+ inet:ipv4=4
6910
+ inet:ipv4=5
6911
+ inet:ipv4=6
6912
+ :asn=4
6913
+
6914
+ (ou:org=*
6915
+ ou:org=*
6916
+ :names=(foo, bar))
6917
+
6918
+ .seen=2020
6919
+ .univarray=(1, 2)
6920
+ +#foo:score=2
6921
+
6922
+ test:arrayform=(3,4,5)
6923
+ test:arrayform=(4,5,6)
6924
+ ]'''
6925
+ await core.nodes(q, opts=forkopts)
6926
+
6927
+ q = 'return($lib.view.get().getPropCount(inet:ipv4:asn))'
6928
+ self.eq(6, await core.callStorm(q, opts=forkopts))
6929
+
6930
+ q = 'return($lib.view.get().getPropCount(inet:ipv4:asn, valu=1))'
6931
+ self.eq(0, await core.callStorm(q, opts=forkopts))
6932
+
6933
+ q = 'return($lib.view.get().getPropCount(inet:ipv4:loc, valu=1))'
6934
+ self.eq(0, await core.callStorm(q, opts=forkopts))
6935
+
6936
+ q = 'return($lib.view.get().getPropCount(inet:ipv4:asn, valu=4))'
6937
+ self.eq(6, await core.callStorm(q, opts=forkopts))
6938
+
6939
+ q = 'return($lib.view.get().getPropCount(inet:ipv4.seen, valu=2020))'
6940
+ self.eq(6, await core.callStorm(q, opts=forkopts))
6941
+
6942
+ q = 'return($lib.view.get().getPropCount(".seen", valu=2020))'
6943
+ self.eq(10, await core.callStorm(q, opts=forkopts))
6944
+
6945
+ q = 'return($lib.view.get().getPropCount(inet:ipv4, valu=1))'
6946
+ self.eq(1, await core.callStorm(q, opts=forkopts))
6947
+
6948
+ q = 'return($lib.view.get().getPropCount(ou:org:names, valu=(foo, bar)))'
6949
+ self.eq(4, await core.callStorm(q, opts=forkopts))
6950
+
6951
+ q = 'return($lib.view.get().getPropCount(".univarray", valu=(1, 2)))'
6952
+ self.eq(10, await core.callStorm(q, opts=forkopts))
6953
+
6954
+ with self.raises(s_exc.NoSuchProp):
6955
+ q = 'return($lib.view.get().getPropCount(newp, valu=1))'
6956
+ await core.callStorm(q, opts=forkopts)
6957
+
6958
+ q = 'return($lib.view.get().getPropArrayCount(ou:org:names))'
6959
+ self.eq(8, await core.callStorm(q, opts=forkopts))
6960
+
6961
+ q = 'return($lib.view.get().getPropArrayCount(ou:org:names, valu=foo))'
6962
+ self.eq(4, await core.callStorm(q, opts=forkopts))
6963
+
6964
+ q = 'return($lib.view.get().getPropArrayCount(".univarray", valu=2))'
6965
+ self.eq(10, await core.callStorm(q, opts=forkopts))
6966
+
6967
+ q = 'return($lib.view.get().getPropArrayCount(test:arrayform, valu=3))'
6968
+ self.eq(3, await core.callStorm(q, opts=forkopts))
6969
+
6970
+ with self.raises(s_exc.NoSuchProp):
6971
+ q = 'return($lib.view.get().getPropArrayCount(newp, valu=1))'
6972
+ await core.callStorm(q, opts=forkopts)
6973
+
6974
+ with self.raises(s_exc.BadTypeValu):
6975
+ q = 'return($lib.view.get().getPropArrayCount(inet:ipv4, valu=1))'
6976
+ await core.callStorm(q, opts=forkopts)
6977
+
6978
+ q = 'return($lib.view.get().getTagPropCount(foo, score))'
6979
+ self.eq(10, await core.callStorm(q, opts=forkopts))
6980
+
6981
+ q = 'return($lib.view.get().getTagPropCount(foo, score, valu=2))'
6982
+ self.eq(10, await core.callStorm(q, opts=forkopts))
6983
+
6984
+ q = 'return($lib.view.get().getTagPropCount(foo, score, form=ou:org, valu=2))'
6985
+ self.eq(4, await core.callStorm(q, opts=forkopts))
6986
+
6987
+ with self.raises(s_exc.NoSuchTagProp):
6988
+ q = 'return($lib.view.get().getTagPropCount(foo, newp, valu=2))'
6989
+ await core.callStorm(q, opts=forkopts)
@@ -168,6 +168,46 @@ class TypesTest(s_t_utils.SynTest):
168
168
  self.eq(node.get('depth'), 2)
169
169
  self.eq(node.get('parent'), 'foo.bar.')
170
170
 
171
+ self.sorteq(
172
+ ['foo.', 'foo.bar.', 'foo.bar.baz.'],
173
+ [n.ndef[1] for n in await core.nodes('test:taxonomy')]
174
+ )
175
+
176
+ self.len(0, await core.nodes('test:taxonomy=foo.bar.ba'))
177
+ self.len(1, await core.nodes('test:taxonomy=foo.bar.baz'))
178
+ self.len(1, await core.nodes('test:taxonomy=foo.bar.baz.'))
179
+
180
+ self.len(0, await core.nodes('test:taxonomy=(foo, bar, ba)'))
181
+ self.len(1, await core.nodes('test:taxonomy=(foo, bar, baz)'))
182
+
183
+ self.len(3, await core.nodes('test:taxonomy^=f'))
184
+ self.len(0, await core.nodes('test:taxonomy^=f.'))
185
+ self.len(2, await core.nodes('test:taxonomy^=foo.b'))
186
+ self.len(0, await core.nodes('test:taxonomy^=foo.b.'))
187
+ self.len(2, await core.nodes('test:taxonomy^=foo.bar'))
188
+ self.len(2, await core.nodes('test:taxonomy^=foo.bar.'))
189
+ self.len(1, await core.nodes('test:taxonomy^=foo.bar.b'))
190
+ self.len(1, await core.nodes('test:taxonomy^=foo.bar.baz'))
191
+ self.len(1, await core.nodes('test:taxonomy^=foo.bar.baz.'))
192
+
193
+ self.len(0, await core.nodes('test:taxonomy^=(f,)'))
194
+ self.len(0, await core.nodes('test:taxonomy^=(foo, b)'))
195
+ self.len(2, await core.nodes('test:taxonomy^=(foo, bar)'))
196
+ self.len(1, await core.nodes('test:taxonomy^=(foo, bar, baz)'))
197
+
198
+ # just test one conditional since RelPropCond and
199
+ # AbsPropCond (for form and prop) use the same logic
200
+ self.len(1, await core.nodes('test:taxonomy:sort=1 +:parent^=f'))
201
+ self.len(0, await core.nodes('test:taxonomy:sort=1 +:parent^=f.'))
202
+ self.len(1, await core.nodes('test:taxonomy:sort=1 +:parent^=foo.b'))
203
+ self.len(0, await core.nodes('test:taxonomy:sort=1 +:parent^=foo.b.'))
204
+ self.len(1, await core.nodes('test:taxonomy:sort=1 +:parent^=foo.bar'))
205
+ self.len(1, await core.nodes('test:taxonomy:sort=1 +:parent^=foo.bar.'))
206
+
207
+ self.len(0, await core.nodes('test:taxonomy:sort=1 +:parent^=(f,)'))
208
+ self.len(0, await core.nodes('test:taxonomy:sort=1 +:parent^=(foo, b)'))
209
+ self.len(1, await core.nodes('test:taxonomy:sort=1 +:parent^=(foo, bar)'))
210
+
171
211
  def test_duration(self):
172
212
  model = s_datamodel.Model()
173
213
  t = model.type('duration')
@@ -147,8 +147,8 @@ class ViewTest(s_t_utils.SynTest):
147
147
  # Add a node back
148
148
  await self.agenlen(1, view2.eval('[ test:int=12 ]'))
149
149
 
150
- # Add a bunch of nodes to require chunking of splices when merging
151
- for i in range(1000):
150
+ # Add a bunch of test nodes to the view.
151
+ for i in range(20):
152
152
  await self.agenlen(1, view2.eval('[test:int=$val]', opts={'vars': {'val': i + 1000}}))
153
153
 
154
154
  # Add prop that will only exist in the child
@@ -217,7 +217,7 @@ class ViewTest(s_t_utils.SynTest):
217
217
  await view2.wipeLayer()
218
218
 
219
219
  # The parent counts includes all the nodes that were merged
220
- self.eq(1005, (await core.view.getFormCounts()).get('test:int'))
220
+ self.eq(25, (await core.view.getFormCounts()).get('test:int'))
221
221
 
222
222
  # A node added to the child is now present in the parent
223
223
  nodes = await core.nodes('test:int=12')
@@ -247,13 +247,15 @@ class ViewTest(s_t_utils.SynTest):
247
247
  self.len(2, await core.nodes('test:int -(refs)> *'))
248
248
 
249
249
  # The child count includes all the nodes in the view
250
- self.eq(1005, (await view2.getFormCounts()).get('test:int'))
250
+ self.eq(25, (await view2.getFormCounts()).get('test:int'))
251
251
 
252
252
  # The child can see nodes that got merged
253
253
  nodes = await view2.nodes('test:int=12')
254
254
  self.len(1, nodes)
255
255
  nodes = await view2.nodes('test:int=1000')
256
256
  self.len(1, nodes)
257
+ nodes = await view2.nodes('test:int=1019')
258
+ self.len(1, nodes)
257
259
 
258
260
  await core.delView(view2.iden)
259
261
  await core.view.addLayer(layriden)
@@ -411,15 +413,6 @@ class ViewTest(s_t_utils.SynTest):
411
413
  self.eq(2, count['node:edits'])
412
414
  self.eq(0, count['node:add'])
413
415
 
414
- mesgs = await core.stormlist('[test:str=foo2 :hehe=bar]', opts={'editformat': 'splices'})
415
- count = collections.Counter(m[0] for m in mesgs)
416
- self.eq(1, count['init'])
417
- self.eq(1, count['node:add'])
418
- self.eq(2, count['prop:set']) # .created and .hehe
419
- self.eq(0, count['node:edits'])
420
- self.eq(1, count['node'])
421
- self.eq(1, count['fini'])
422
-
423
416
  mesgs = await core.stormlist('[test:str=foo3 :hehe=bar]', opts={'editformat': 'count'})
424
417
  count = collections.Counter(m[0] for m in mesgs)
425
418
  self.eq(1, count['init'])
@@ -243,7 +243,7 @@ class BaseTest(s_t_utils.SynTest):
243
243
  r2 = await snap.addNode('edge:refs', (cnode.ndef, ('test:int', 1234)))
244
244
 
245
245
  # Gather up all the nodes in the cluster
246
- nodes = await snap.eval(f'graph:cluster={guid} -+> edge:refs -+> * | uniq').list()
246
+ nodes = await core.nodes(f'graph:cluster={guid} -+> edge:refs -+> * | uniq')
247
247
  self.len(5, nodes)
248
248
 
249
249
  async def test_model_base_rules(self):
@@ -953,6 +953,12 @@ class InetModelTest(s_t_utils.SynTest):
953
953
  with self.raises(s_exc.BadTypeValu):
954
954
  await core.nodes('[inet:ipv4=foo-bar-duck.com]')
955
955
 
956
+ with self.raises(s_exc.BadTypeValu):
957
+ await core.nodes('[inet:ipv4=3.87/nice/index.php]')
958
+
959
+ with self.raises(s_exc.BadTypeValu):
960
+ await core.nodes('[inet:ipv4=3.87/33]')
961
+
956
962
  with self.raises(s_exc.BadTypeValu):
957
963
  await core.nodes('[test:str="foo"] [inet:ipv4=$node.value()]')
958
964
 
@@ -2778,3 +2784,12 @@ class InetModelTest(s_t_utils.SynTest):
2778
2784
  self.eq(nodes[0].get('client'), 'tcp://1.2.3.4')
2779
2785
  self.eq(nodes[0].get('client:ipv4'), 0x01020304)
2780
2786
  self.eq(nodes[0].get('client:ipv6'), '::1')
2787
+
2788
+ async def test_model_inet_onset_depth(self):
2789
+
2790
+ async with self.getTestCore() as core:
2791
+ fqdn = '.'.join(['x' for x in range(150)]) + '.foo.com'
2792
+ q = f'[ inet:fqdn="{fqdn}"]'
2793
+ nodes = await core.nodes(q)
2794
+ self.len(1, nodes)
2795
+ self.eq(nodes[0].get('zone'), 'foo.com')
@@ -164,6 +164,34 @@ class InfotechModelTest(s_t_utils.SynTest):
164
164
  self.eq(nodes[0].get('addresses'), ('T0100', 'T0200'))
165
165
  self.eq(nodes[0].get('matrix'), 'enterprise')
166
166
 
167
+ nodes = await core.nodes('''[
168
+ it:mitre:attack:campaign=C0001
169
+ :created = 20231101
170
+ :desc = "Much campaign, many sophisticated."
171
+ :groups = (G0100,)
172
+ :matrices = (enterprise,ics)
173
+ :name = "much campaign"
174
+ :names = ('much campaign', 'many sophisticated')
175
+ :software = (S0100,)
176
+ :techniques = (T0200,T0100)
177
+ :updated = 20231102
178
+ :url = https://attack.mitre.org/campaigns/C0001
179
+ :period = (20151201, 20160101)
180
+ ]''')
181
+ self.len(1, nodes)
182
+ self.eq(nodes[0].ndef, ('it:mitre:attack:campaign', 'C0001'))
183
+ self.eq(nodes[0].props.get('name'), 'much campaign')
184
+ self.eq(nodes[0].props.get('names'), ('many sophisticated', 'much campaign'))
185
+ self.eq(nodes[0].props.get('desc'), 'Much campaign, many sophisticated.')
186
+ self.eq(nodes[0].props.get('url'), 'https://attack.mitre.org/campaigns/C0001')
187
+ self.eq(nodes[0].props.get('matrices'), ('enterprise', 'ics'))
188
+ self.eq(nodes[0].props.get('groups'), ('G0100',))
189
+ self.eq(nodes[0].props.get('software'), ('S0100',))
190
+ self.eq(nodes[0].props.get('techniques'), ('T0100', 'T0200'))
191
+ self.eq(nodes[0].props.get('created'), 1698796800000)
192
+ self.eq(nodes[0].props.get('updated'), 1698883200000)
193
+ self.eq(nodes[0].props.get('period'), (1448928000000, 1451606400000))
194
+
167
195
  nodes = await core.nodes('''[
168
196
  it:exec:thread=*
169
197
  :proc=*
@@ -284,6 +312,58 @@ class InfotechModelTest(s_t_utils.SynTest):
284
312
  self.len(1, await core.nodes('it:app:yara:procmatch -> it:exec:proc'))
285
313
  self.len(1, await core.nodes('it:app:yara:procmatch -> it:app:yara:rule'))
286
314
 
315
+ nodes = await core.nodes('''[
316
+ it:av:scan:result=*
317
+ :time=20231117
318
+ :verdict=suspicious
319
+ :scanner={[ it:prod:softver=* :name="visi scan" ]}
320
+ :scanner:name="visi scan"
321
+ :signame=omgwtfbbq
322
+ :target:file=*
323
+ :target:proc={[ it:exec:proc=* :cmd="foo.exe --bar" ]}
324
+ :target:host={[ it:host=* :name=visihost ]}
325
+ :target:fqdn=vertex.link
326
+ :target:url=https://vertex.link
327
+ :target:ipv4=1.2.3.4
328
+ :target:ipv6='::1'
329
+ :multi:scan={[ it:av:scan:result=*
330
+ :scanner:name="visi total"
331
+ :multi:count=10
332
+ :multi:count:benign=3
333
+ :multi:count:unknown=1
334
+ :multi:count:suspicious=4
335
+ :multi:count:malicious=2
336
+ ]}
337
+ ]''')
338
+ self.eq(1700179200000, nodes[0].get('time'))
339
+ self.eq(30, nodes[0].get('verdict'))
340
+ self.eq('visi scan', nodes[0].get('scanner:name'))
341
+ self.eq('vertex.link', nodes[0].get('target:fqdn'))
342
+ self.eq('https://vertex.link', nodes[0].get('target:url'))
343
+ self.eq(0x01020304, nodes[0].get('target:ipv4'))
344
+ self.eq('::1', nodes[0].get('target:ipv6'))
345
+ self.eq('omgwtfbbq', nodes[0].get('signame'))
346
+
347
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> it:host'))
348
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> inet:url'))
349
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> inet:fqdn'))
350
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> file:bytes'))
351
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> it:exec:proc'))
352
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> it:av:signame'))
353
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> it:prod:softver'))
354
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi scan" -> it:prod:softname'))
355
+
356
+ nodes = await core.nodes('it:av:scan:result:scanner:name="visi total"')
357
+ self.len(1, nodes)
358
+
359
+ self.eq(10, nodes[0].get('multi:count'))
360
+ self.eq(3, nodes[0].get('multi:count:benign'))
361
+ self.eq(1, nodes[0].get('multi:count:unknown'))
362
+ self.eq(4, nodes[0].get('multi:count:suspicious'))
363
+ self.eq(2, nodes[0].get('multi:count:malicious'))
364
+
365
+ self.len(1, await core.nodes('it:av:scan:result:scanner:name="visi total" -> it:av:scan:result +:scanner:name="visi scan"'))
366
+
287
367
  async def test_infotech_ios(self):
288
368
 
289
369
  async with self.getTestCore() as core:
@@ -436,6 +516,34 @@ class InfotechModelTest(s_t_utils.SynTest):
436
516
  self.eq(7260000, nodes[1].get('duration'))
437
517
  self.eq('02:01:00.000', nodes[1].repr('duration'))
438
518
 
519
+ # Sample SIDs from here:
520
+ # https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/81d92bba-d22b-4a8c-908a-554ab29148ab
521
+ sids = [
522
+ 'S-1-0-0', 'S-1-1-0', 'S-1-2-0', 'S-1-2-1', 'S-1-3', 'S-1-3-0',
523
+ 'S-1-3-1', 'S-1-3-2', 'S-1-3-3', 'S-1-3-4', 'S-1-5', 'S-1-5-1',
524
+ 'S-1-5-2', 'S-1-5-3', 'S-1-5-4', 'S-1-5-6', 'S-1-5-7', 'S-1-5-8',
525
+ 'S-1-5-9', 'S-1-5-10', 'S-1-5-11', 'S-1-5-12', 'S-1-5-13', 'S-1-5-14',
526
+ 'S-1-5-15', 'S-1-5-17', 'S-1-5-18', 'S-1-5-19', 'S-1-5-20',
527
+ 'S-1-5-21-0-0-0-496', 'S-1-5-21-0-0-0-497', 'S-1-5-32-544',
528
+ 'S-1-5-32-545', 'S-1-5-32-546', 'S-1-5-32-547', 'S-1-5-32-548',
529
+ 'S-1-5-32-549', 'S-1-5-32-550', 'S-1-5-32-551', 'S-1-5-32-552',
530
+ 'S-1-5-32-554', 'S-1-5-32-555', 'S-1-5-32-556', 'S-1-5-32-557',
531
+ 'S-1-5-32-558', 'S-1-5-32-559', 'S-1-5-32-560', 'S-1-5-32-561',
532
+ 'S-1-5-32-562', 'S-1-5-32-568', 'S-1-5-32-569', 'S-1-5-32-573',
533
+ 'S-1-5-32-574', 'S-1-5-32-575', 'S-1-5-32-576', 'S-1-5-32-577',
534
+ 'S-1-5-32-578', 'S-1-5-32-579', 'S-1-5-32-580', 'S-1-5-32-582',
535
+ 'S-1-5-33', 'S-1-5-64-10', 'S-1-5-64-14', 'S-1-5-64-21', 'S-1-5-65-1',
536
+ 'S-1-5-80', 'S-1-5-80-0', 'S-1-5-83-0', 'S-1-5-84-0-0-0-0-0',
537
+ 'S-1-5-90-0', 'S-1-5-113', 'S-1-5-114', 'S-1-5-1000', 'S-1-15-2-1',
538
+ 'S-1-16-0', 'S-1-16-4096', 'S-1-16-8192', 'S-1-16-8448', 'S-1-16-12288',
539
+ 'S-1-16-16384', 'S-1-16-20480', 'S-1-16-28672', 'S-1-18-1', 'S-1-18-2',
540
+ 'S-1-18-3', 'S-1-18-4', 'S-1-18-5', 'S-1-18-6',
541
+ ]
542
+
543
+ opts = {'vars': {'sids': sids}}
544
+ nodes = await core.nodes('for $sid in $sids {[ it:account=* :windows:sid=$sid ]}', opts=opts)
545
+ self.len(88, nodes)
546
+
439
547
  nodes = await core.nodes('inet:email=visi@vertex.link -> ps:contact -> it:account -> it:logon +:time>=2021 -> it:host')
440
548
  self.len(1, nodes)
441
549
  self.eq('it:host', nodes[0].ndef[0])
@@ -1360,10 +1468,12 @@ class InfotechModelTest(s_t_utils.SynTest):
1360
1468
  :opts=({"foo": "bar"})
1361
1469
  :api:url=https://vertex.link/api/v1.
1362
1470
  :time=20220720
1471
+ :offset=99
1363
1472
  // we can assume the rest of the interface props work
1364
1473
  ]
1365
1474
  ''')
1366
1475
  self.eq(1658275200000, nodes[0].get('time'))
1476
+ self.eq(99, nodes[0].get('offset'))
1367
1477
  self.eq('sql', nodes[0].get('language'))
1368
1478
  self.eq({"foo": "bar"}, nodes[0].get('opts'))
1369
1479
  self.eq('SELECT * FROM threats', nodes[0].get('text'))
@@ -68,6 +68,10 @@ class OuModelTest(s_t_utils.SynTest):
68
68
 
69
69
  timeline = s_common.guid()
70
70
 
71
+ nodes = await core.nodes('[ it:mitre:attack:campaign=C0011 ]')
72
+ self.len(1, nodes)
73
+ attack = nodes[0].ndef
74
+
71
75
  props = {
72
76
  'org': org0,
73
77
  'goal': goal,
@@ -85,6 +89,7 @@ class OuModelTest(s_t_utils.SynTest):
85
89
  'reporter': '*',
86
90
  'reporter:name': 'vertex',
87
91
  'timeline': timeline,
92
+ 'mitre:attack:campaign': 'C0011',
88
93
  }
89
94
  node = await snap.addNode('ou:campaign', camp, props=props)
90
95
  self.eq(node.get('tag'), 'cno.camp.31337')
@@ -101,10 +106,15 @@ class OuModelTest(s_t_utils.SynTest):
101
106
  self.eq(node.get('camptype'), 'get.pizza.')
102
107
  self.eq(node.get('techniques'), tuple(sorted(teqs)))
103
108
  self.eq(node.get('timeline'), timeline)
109
+ self.eq(node.get('mitre:attack:campaign'), 'C0011')
104
110
 
105
111
  self.nn(node.get('reporter'))
106
112
  self.eq(node.get('reporter:name'), 'vertex')
107
113
 
114
+ nodes = await core.nodes(f'ou:campaign={camp} -> it:mitre:attack:campaign')
115
+ self.len(1, nodes)
116
+ self.eq(nodes[0].ndef, attack)
117
+
108
118
  # type norming first
109
119
  # ou:name
110
120
  t = core.model.type('ou:name')
@@ -67,8 +67,6 @@ class PsModelTest(s_t_utils.SynTest):
67
67
  self.eq(node.get('name:given'), 'эммануэль')
68
68
  self.eq(node.get('nicks'), ['beeper88', 'w1ntermut3'])
69
69
  self.eq(node.get('names'), ['bob ross'])
70
- # self.eq(node.get('img'), '') # fixme file:bytes
71
- # self.eq(node.get('guidname'), '') # fixme guid aliases
72
70
 
73
71
  node = await snap.addNode('ps:person:has', (person0, ('test:str', 'sewer map')))
74
72
  self.eq(node.ndef[1], (person0, ('test:str', 'sewer map')))
@@ -94,7 +92,6 @@ class PsModelTest(s_t_utils.SynTest):
94
92
  'place': place,
95
93
  'place:name': 'The Shire',
96
94
  'orgname': 'Stark Industries, INC',
97
- # 'img': '', # fixme file:bytes
98
95
  'user': 'ironman',
99
96
  'web:acct': ('twitter.com', 'ironman'),
100
97
  'web:group': ('twitter.com', 'avengers'),
@@ -236,13 +236,14 @@ class ProjModelTest(s_test.SynTest):
236
236
  self.len(1, nodes)
237
237
  self.eq(proj, nodes[0].get('project'))
238
238
 
239
- nodes = await core.nodes('proj:ticket [ :ext:creator={[ps:contact=* :name=visi ]} ]')
239
+ nodes = await core.nodes('proj:ticket [ :ext:creator={[ps:contact=* :name=visi ]} :ext:assignee={[ps:contact=* :name=bob ]} ]')
240
240
  self.len(1, nodes)
241
241
  self.nn(nodes[0].get('creator'))
242
242
  self.nn(nodes[0].get('created'))
243
243
  self.nn(nodes[0].get('updated'))
244
244
  self.nn(nodes[0].get('assignee'))
245
245
  self.nn(nodes[0].get('ext:creator'))
246
+ self.nn(nodes[0].get('ext:assignee'))
246
247
  self.eq(70, nodes[0].get('status'))
247
248
  self.eq(50, nodes[0].get('priority'))
248
249
  self.eq('done', nodes[0].repr('status'))