synapse 2.155.0__py311-none-any.whl → 2.156.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (64) hide show
  1. synapse/cmds/cortex.py +2 -14
  2. synapse/common.py +1 -28
  3. synapse/cortex.py +10 -510
  4. synapse/lib/ast.py +60 -1
  5. synapse/lib/cell.py +33 -8
  6. synapse/lib/certdir.py +11 -0
  7. synapse/lib/cmdr.py +0 -5
  8. synapse/lib/gis.py +2 -2
  9. synapse/lib/httpapi.py +1 -43
  10. synapse/lib/layer.py +64 -201
  11. synapse/lib/lmdbslab.py +11 -0
  12. synapse/lib/node.py +1 -3
  13. synapse/lib/parser.py +10 -0
  14. synapse/lib/snap.py +121 -21
  15. synapse/lib/storm.lark +23 -6
  16. synapse/lib/storm.py +15 -338
  17. synapse/lib/storm_format.py +5 -0
  18. synapse/lib/stormlib/gen.py +1 -2
  19. synapse/lib/stormlib/gis.py +41 -0
  20. synapse/lib/stormlib/stats.py +21 -2
  21. synapse/lib/stormlib/storm.py +16 -1
  22. synapse/lib/stormtypes.py +225 -12
  23. synapse/lib/version.py +2 -2
  24. synapse/lib/view.py +96 -21
  25. synapse/models/inet.py +60 -30
  26. synapse/models/infotech.py +56 -1
  27. synapse/models/orgs.py +3 -0
  28. synapse/models/risk.py +15 -0
  29. synapse/models/syn.py +0 -38
  30. synapse/tests/test_cmds_cortex.py +1 -1
  31. synapse/tests/test_cortex.py +32 -336
  32. synapse/tests/test_lib_agenda.py +19 -54
  33. synapse/tests/test_lib_aha.py +97 -0
  34. synapse/tests/test_lib_ast.py +402 -0
  35. synapse/tests/test_lib_grammar.py +30 -10
  36. synapse/tests/test_lib_httpapi.py +0 -46
  37. synapse/tests/test_lib_layer.py +19 -234
  38. synapse/tests/test_lib_lmdbslab.py +22 -0
  39. synapse/tests/test_lib_snap.py +9 -0
  40. synapse/tests/test_lib_storm.py +16 -309
  41. synapse/tests/test_lib_stormlib_gis.py +21 -0
  42. synapse/tests/test_lib_stormlib_stats.py +107 -20
  43. synapse/tests/test_lib_stormlib_storm.py +25 -0
  44. synapse/tests/test_lib_stormtypes.py +231 -8
  45. synapse/tests/test_lib_view.py +6 -13
  46. synapse/tests/test_model_base.py +1 -1
  47. synapse/tests/test_model_inet.py +15 -0
  48. synapse/tests/test_model_infotech.py +60 -0
  49. synapse/tests/test_model_orgs.py +10 -0
  50. synapse/tests/test_model_person.py +0 -3
  51. synapse/tests/test_model_risk.py +20 -0
  52. synapse/tests/test_model_syn.py +20 -34
  53. synapse/tests/test_tools_csvtool.py +2 -1
  54. synapse/tests/test_tools_feed.py +4 -30
  55. synapse/tools/csvtool.py +2 -1
  56. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/METADATA +3 -3
  57. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/RECORD +60 -62
  58. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/WHEEL +1 -1
  59. synapse/cmds/cron.py +0 -726
  60. synapse/cmds/trigger.py +0 -319
  61. synapse/tests/test_cmds_cron.py +0 -453
  62. synapse/tests/test_cmds_trigger.py +0 -176
  63. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/LICENSE +0 -0
  64. {synapse-2.155.0.dist-info → synapse-2.156.0.dist-info}/top_level.txt +0 -0
@@ -685,6 +685,16 @@ Queries = [
685
685
  'reverse(inet:ipv4)',
686
686
  'reverse(inet:ipv4=1.2.3.4)',
687
687
  'reverse(*$form=$valu)',
688
+ 'inet:ipv4 --+> *',
689
+ 'file:bytes <+-- *',
690
+ 'inet:asn <+("edge")- *',
691
+ 'inet:asn -("edge")+> *',
692
+ 'file:bytes -(($foobar, $bizbaz))+> ($biz, $boz)=lol',
693
+ 'media:news <+((neato, burrito))- inet:fqdn',
694
+ 'inet:ipv4 <+(*)- media:news',
695
+ 'media:news -(*)+> inet:fqdn',
696
+ 'inet:ipv4 <+(*)- *',
697
+ 'media:news -(*)+> *'
688
698
  ]
689
699
 
690
700
  # Generated with print_parse_list below
@@ -704,7 +714,7 @@ _ParseResults = [
704
714
  'Query: [EditTagPropSet: [TagProp: [TagName: [VarValue: [Const: tag]], VarValue: [Const: tagprop]], Const: =, VarValue: [Const: valu]]]',
705
715
  'Query: [VarEvalOper: [FuncCall: [VarDeref: [VarValue: [Const: lib], Const: print], CallArgs: [FormatString: [Const: ip=, FuncCall: [VarDeref: [VarValue: [Const: node], Const: repr], CallArgs: [], CallKwargs: []], Const: asn=, RelPropValue: [Const: asn], Const: .seen=, UnivPropValue: [Const: .seen], Const: foo=, TagValue: [TagName: [Const: foo]], Const: , ExprNode: [RelPropValue: [Const: asn], Const: =, Const: 5]]], CallKwargs: []]]]',
706
716
  'Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: 45.79.131.138], FormPivot: [AbsProp: inet:flow], isjoin=False, FiltOper: [Const: -, OrCond: [RelPropCond: [RelPropValue: [RelProp: [Const: dst:port]], Const: =, Const: 444], RelPropCond: [RelPropValue: [RelProp: [Const: dst:port]], Const: =, Const: 80]]]]',
707
- 'Query: [LiftPropBy: [Const: media:news, Const: =, Const: 0c7f7267d3b62432cb0d7d0e9d3108a4], N1Walk: [Const: refs, Const: inet:ipv4]]',
717
+ 'Query: [LiftPropBy: [Const: media:news, Const: =, Const: 0c7f7267d3b62432cb0d7d0e9d3108a4], N1Walk: [Const: refs, Const: inet:ipv4], isjoin=False]',
708
718
  'Query: [SetVarOper: [Const: foo, DollarExpr: [Const: 2]], Return: [DollarExpr: [ExprNode: [VarValue: [Const: foo], Const: +, DollarExpr: [Const: 1]]]]]',
709
719
  'Query: [SetVarOper: [Const: foo, DollarExpr: [Const: 2]], Return: [DollarExpr: [ExprNode: [VarValue: [Const: foo], Const: -, DollarExpr: [Const: 1]]]]]',
710
720
  'Query: [Return: [DollarExpr: [ExprNode: [DollarExpr: [Const: 2], Const: +, DollarExpr: [Const: 1]]]]]',
@@ -722,8 +732,8 @@ _ParseResults = [
722
732
  'Query: [SetVarOper: [Const: p, Const: bar], EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4], EditTagAdd: [TagName: [Const: foo, Const: bar], Const: =, DollarExpr: [Const: 0]], Return: [DollarExpr: [ExprNode: [TagValue: [TagName: [Const: foo, VarValue: [Const: p]]], Const: <, ExprNode: [DollarExpr: [Const: 4], Const: -, DollarExpr: [Const: 1]]]]]]',
723
733
  'Query: [EditNodeAdd: [FormName: [Const: inet:asn], Const: =, Const: 1], EditNodeAdd: [FormName: [Const: inet:asn], Const: =, Const: 2], EditEdgeAdd: [Const: foo), SubQuery: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: 1.2.3.4]]]]]',
724
734
  'Query: [LiftPropBy: [Const: inet:asn, Const: <, DollarExpr: [Const: 2]]]',
725
- 'Query: [LiftProp: [Const: inet:asn], N2Walk: [Const: foo), Const: *]]',
726
- 'Query: [LiftProp: [Const: inet:asn], N2Walk: [List: [Const: foo), Const: bar()], Const: *]]',
735
+ 'Query: [LiftProp: [Const: inet:asn], N2Walk: [Const: foo), Const: *], isjoin=False]',
736
+ 'Query: [LiftProp: [Const: inet:asn], N2Walk: [List: [Const: foo), Const: bar()], Const: *], isjoin=False]',
727
737
  'Query: [Emit: [VarValue: [Const: foo]], Stop: []]',
728
738
  'Query: [TryCatch: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: asdf]], CatchBlock: [Const: TypeError, Const: err, Query: []]]]',
729
739
  'Query: [TryCatch: [Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: asdf]], CatchBlock: [Const: FooBar, Const: err, Query: []], CatchBlock: [Const: *, Const: err, Query: []]]]',
@@ -739,10 +749,10 @@ _ParseResults = [
739
749
  'Query: [LiftProp: [Const: inet:ipv4], N2WalkNPivo: [], isjoin=False]',
740
750
  'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: woot.com], EditEdgeAdd: [Const: refs, SubQuery: [Query: [LiftProp: [Const: media:news]]]]]',
741
751
  'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: woot.com], EditEdgeAdd: [Const: refs, SubQuery: [Query: [LiftProp: [Const: media:news]]]]]',
742
- 'Query: [SetVarOper: [Const: refs, Const: refs], LiftProp: [Const: media:news], N1Walk: [VarValue: [Const: refs], Const: *], FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
743
- 'Query: [SetVarOper: [Const: refs, Const: refs], LiftProp: [Const: media:news], N2Walk: [VarValue: [Const: refs], List: [Const: inet:ipv4, Const: inet:ipv6]], FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
744
- 'Query: [LiftProp: [Const: media:news], N1Walk: [Const: refs, Const: *], FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
745
- 'Query: [LiftProp: [Const: media:news], N2Walk: [Const: refs, VarValue: [Const: bar]], FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
752
+ 'Query: [SetVarOper: [Const: refs, Const: refs], LiftProp: [Const: media:news], N1Walk: [VarValue: [Const: refs], Const: *], isjoin=False, FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
753
+ 'Query: [SetVarOper: [Const: refs, Const: refs], LiftProp: [Const: media:news], N2Walk: [VarValue: [Const: refs], List: [Const: inet:ipv4, Const: inet:ipv6]], isjoin=False, FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
754
+ 'Query: [LiftProp: [Const: media:news], N1Walk: [Const: refs, Const: *], isjoin=False, FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
755
+ 'Query: [LiftProp: [Const: media:news], N2Walk: [Const: refs, VarValue: [Const: bar]], isjoin=False, FiltOper: [Const: -, OrCond: [TagCond: [TagMatch: [Const: foo]], TagCond: [TagMatch: [Const: bar]]]]]',
746
756
  'Query: [LiftProp: [Const: media:news], EditEdgeDel: [Const: refs, SubQuery: [Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: woot.com]]]]]',
747
757
  'Query: [LiftProp: [Const: media:news], EditEdgeAdd: [Const: refs, SubQuery: [Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: woot.com]]]]]',
748
758
  'Query: [CmdOper: [Const: cron, List: [Const: add, Const: --monthly, Const: -1:12:30, ArgvQuery: [Query: [LiftTag: [TagName: [Const: bar]]]]]]]',
@@ -1237,9 +1247,9 @@ _ParseResults = [
1237
1247
  'Query: [LiftProp: [Const: inet:ipv4], FiltOper: [Const: +, DollarExpr: [ExprAndNode: [Const: 0, Const: and, Const: 1]]]]',
1238
1248
  'Query: [SetVarOper: [Const: x, DollarExpr: [ExprNode: [VarValue: [Const: x], Const: -, Const: 1]]]]',
1239
1249
  'Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: 1.2.3.4], FiltOper: [Const: +, DollarExpr: [ExprNode: [ExprNode: [RelPropValue: [Const: asn], Const: +, Const: 20], Const: >=, Const: 42]]]]',
1240
- 'Query: [LiftProp: [Const: inet:ipv4], N1Walk: [Const: seen, Const: foo:bar:baz]]',
1241
- 'Query: [LiftProp: [Const: inet:ipv4], N1Walk: [Const: seen, List: [Const: foo:bar:baz, Const: hehe:haha:hoho], Const: ^=, Const: lol]]',
1242
- "Query: [LiftProp: [Const: inet:ipv4], N1Walk: [VarList: ['foo', 'bar'], List: [VarValue: [Const: baz], VarValue: [Const: faz]], Const: =, Const: lol]]",
1250
+ 'Query: [LiftProp: [Const: inet:ipv4], N1Walk: [Const: seen, Const: foo:bar:baz], isjoin=False]',
1251
+ 'Query: [LiftProp: [Const: inet:ipv4], N1Walk: [Const: seen, List: [Const: foo:bar:baz, Const: hehe:haha:hoho], Const: ^=, Const: lol], isjoin=False]',
1252
+ "Query: [LiftProp: [Const: inet:ipv4], N1Walk: [VarList: ['foo', 'bar'], List: [VarValue: [Const: baz], VarValue: [Const: faz]], Const: =, Const: lol], isjoin=False]",
1243
1253
  'Query: [SetVarOper: [Const: x, DollarExpr: [ExprList: [Const: foo, Const: bar]]]]',
1244
1254
  'Query: [SetVarOper: [Const: x, DollarExpr: [ExprList: [Const: foo, Const: bar]]]]',
1245
1255
  'Query: [SetVarOper: [Const: x, DollarExpr: [ExprDict: [Const: foo, Const: bar, Const: baz, Const: 10]]]]',
@@ -1274,6 +1284,16 @@ _ParseResults = [
1274
1284
  'Query: [LiftProp: [Const: inet:ipv4]]',
1275
1285
  'Query: [LiftPropBy: [Const: inet:ipv4, Const: =, Const: 1.2.3.4]]',
1276
1286
  'Query: [LiftPropBy: [VarValue: [Const: form], Const: =, VarValue: [Const: valu]]]',
1287
+ 'Query: [LiftProp: [Const: inet:ipv4], N1WalkNPivo: [], isjoin=True]',
1288
+ 'Query: [LiftProp: [Const: file:bytes], N2WalkNPivo: [], isjoin=True]',
1289
+ 'Query: [LiftProp: [Const: inet:asn], N2Walk: [Const: edge, Const: *], isjoin=True]',
1290
+ 'Query: [LiftProp: [Const: inet:asn], N1Walk: [Const: edge, Const: *], isjoin=True]',
1291
+ "Query: [LiftProp: [Const: file:bytes], N1Walk: [VarList: ['foobar', 'bizbaz'], List: [VarValue: [Const: biz], VarValue: [Const: boz]], Const: =, Const: lol], isjoin=True]",
1292
+ 'Query: [LiftProp: [Const: media:news], N2Walk: [List: [Const: neato, Const: burrito], Const: inet:fqdn], isjoin=True]',
1293
+ 'Query: [LiftProp: [Const: inet:ipv4], N2Walk: [Const: *, Const: media:news], isjoin=True]',
1294
+ 'Query: [LiftProp: [Const: media:news], N1Walk: [Const: *, Const: inet:fqdn], isjoin=True]',
1295
+ 'Query: [LiftProp: [Const: inet:ipv4], N2Walk: [Const: *, Const: *], isjoin=True]',
1296
+ 'Query: [LiftProp: [Const: media:news], N1Walk: [Const: *, Const: *], isjoin=True]',
1277
1297
  ]
1278
1298
 
1279
1299
  class GrammarTest(s_t_utils.SynTest):
@@ -797,52 +797,6 @@ class HttpApiTest(s_tests.SynTest):
797
797
  retn = await resp.json()
798
798
  self.eq('err', retn.get('status'))
799
799
 
800
- async def test_http_watch(self):
801
-
802
- async with self.getTestCore() as core:
803
-
804
- visi = await core.auth.addUser('visi')
805
-
806
- await visi.setPasswd('secret')
807
-
808
- host, port = await core.addHttpsPort(0, host='127.0.0.1')
809
-
810
- # with no session user...
811
- async with self.getHttpSess() as sess:
812
-
813
- async with sess.ws_connect(f'wss://localhost:{port}/api/v1/watch') as sock:
814
- await sock.send_json({'tags': ['test.visi']})
815
- mesg = await sock.receive_json()
816
- self.eq('errx', mesg['type'])
817
- self.eq('AuthDeny', mesg['data']['code'])
818
-
819
- async with sess.post(f'https://localhost:{port}/api/v1/login', json={'user': 'visi', 'passwd': 'secret'}) as resp:
820
- retn = await resp.json()
821
- self.eq('ok', retn.get('status'))
822
- self.eq('visi', retn['result']['name'])
823
-
824
- async with sess.ws_connect(f'wss://localhost:{port}/api/v1/watch') as sock:
825
- await sock.send_json({'tags': ['test.visi']})
826
- mesg = await sock.receive_json()
827
- self.eq('errx', mesg['type'])
828
- self.eq('AuthDeny', mesg['data']['code'])
829
-
830
- await visi.addRule((True, ('watch',)))
831
-
832
- async with sess.ws_connect(f'wss://localhost:{port}/api/v1/watch') as sock:
833
-
834
- await sock.send_json({'tags': ['test.visi']})
835
- mesg = await sock.receive_json()
836
-
837
- self.eq('init', mesg['type'])
838
-
839
- await core.nodes('[ test:str=woot +#test.visi ]')
840
-
841
- mesg = await sock.receive_json()
842
-
843
- self.eq('tag:add', mesg['type'])
844
- self.eq('test.visi', mesg['data']['tag'])
845
-
846
800
  async def test_http_beholder(self):
847
801
  self.skipIfNexusReplay()
848
802
  async with self.getTestCore() as core:
@@ -456,16 +456,15 @@ class LayerTest(s_t_utils.SynTest):
456
456
 
457
457
  self.len(1, await core01.nodes('inet:ipv4=5.6.7.8'))
458
458
 
459
- # make sure time and user are set on the downstream splices
459
+ # make sure time and user are set on the downstream changes
460
460
  root = await core01.auth.getUserByName('root')
461
461
 
462
- splices = await alist(layr.splicesBack(size=1))
463
- self.len(1, splices)
464
-
465
- splice = splices[0][1][1]
466
- self.nn(splice.get('time'))
467
- self.eq(splice.get('user'), root.iden)
468
- self.none(splice.get('prov'))
462
+ nedits = await alist(layr.syncNodeEdits2(0, wait=False))
463
+ last_edit = nedits[-1]
464
+ offs, edit, meta = last_edit
465
+ self.gt(meta.get('time'), 0)
466
+ self.eq(meta.get('user'), root.iden)
467
+ self.notin('prov', meta)
469
468
 
470
469
  async def test_layer_upstream_with_mirror(self):
471
470
 
@@ -626,223 +625,6 @@ class LayerTest(s_t_utils.SynTest):
626
625
 
627
626
  self.len(1, await core02.nodes('inet:ipv4=8.7.6.5'))
628
627
 
629
- async def test_layer_splices(self):
630
-
631
- self.skip('fragile order and deprecated')
632
- async with self.getTestCore() as core:
633
-
634
- layr = core.view.layers[0]
635
- root = await core.auth.getUserByName('root')
636
-
637
- splices = await alist(layr.splices(None, 10))
638
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
639
-
640
- await core.addTagProp('risk', ('int', {'min': 0, 'max': 100}), {'doc': 'risk score'})
641
-
642
- # Convert a node:add splice
643
- await core.nodes('[ test:str=foo ]')
644
-
645
- splices = await alist(layr.splices(spliceoffs, 10))
646
-
647
- splice = splices[0][1]
648
- self.eq(splice[0], 'node:add')
649
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
650
- self.eq(splice[1]['user'], root.iden)
651
- self.nn(splice[1].get('time'))
652
-
653
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
654
-
655
- # Convert a prop:set splice with no oldv
656
- await core.nodes("test:str=foo [ :tick=2000 ]")
657
-
658
- splices = await alist(layr.splices(spliceoffs, 10))
659
-
660
- splice = splices[0][1]
661
- self.eq(splice[0], 'prop:set')
662
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
663
- self.eq(splice[1]['prop'], 'tick')
664
- self.eq(splice[1]['valu'], 946684800000)
665
- self.eq(splice[1]['oldv'], None)
666
- self.eq(splice[1]['user'], root.iden)
667
- self.nn(splice[1].get('time'))
668
-
669
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
670
-
671
- # Convert a prop:set splice with an oldv
672
- await core.nodes("test:str=foo [ :tick=2001 ]")
673
-
674
- splices = await alist(layr.splices(spliceoffs, 10))
675
-
676
- splice = splices[0][1]
677
- self.eq(splice[0], 'prop:set')
678
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
679
- self.eq(splice[1]['prop'], 'tick')
680
- self.eq(splice[1]['valu'], 978307200000)
681
- self.eq(splice[1]['oldv'], 946684800000)
682
- self.eq(splice[1]['user'], root.iden)
683
- self.nn(splice[1].get('time'))
684
-
685
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
686
-
687
- # Convert a prop:del splice
688
- await core.nodes("test:str=foo [ -:tick ]")
689
-
690
- splices = await alist(layr.splices(spliceoffs, 10))
691
-
692
- splice = splices[0][1]
693
- self.eq(splice[0], 'prop:del')
694
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
695
- self.eq(splice[1]['prop'], 'tick')
696
- self.eq(splice[1]['valu'], 978307200000)
697
- self.eq(splice[1]['user'], root.iden)
698
- self.nn(splice[1].get('time'))
699
-
700
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
701
-
702
- # Convert a tag:add splice with no oldv
703
- await core.nodes("test:str=foo [ +#haha=2000 ]")
704
-
705
- splices = await alist(layr.splices(spliceoffs, 10))
706
-
707
- splice = splices[4][1]
708
- self.eq(splice[0], 'tag:add')
709
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
710
- self.eq(splice[1]['tag'], 'haha')
711
- self.eq(splice[1]['valu'], (946684800000, 946684800001))
712
- self.eq(splice[1]['oldv'], None)
713
- self.eq(splice[1]['user'], root.iden)
714
- self.nn(splice[1].get('time'))
715
-
716
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
717
-
718
- # Convert a tag:add splice with an oldv
719
- await core.nodes("test:str=foo [ +#haha=2001 ]")
720
-
721
- splices = await alist(layr.splices(spliceoffs, 10))
722
-
723
- splice = splices[0][1]
724
- self.eq(splice[0], 'tag:add')
725
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
726
- self.eq(splice[1]['tag'], 'haha')
727
- self.eq(splice[1]['valu'], (946684800000, 978307200001))
728
- self.eq(splice[1]['oldv'], (946684800000, 946684800001))
729
- self.eq(splice[1]['user'], root.iden)
730
- self.nn(splice[1].get('time'))
731
-
732
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
733
-
734
- # Convert a tag:del splice
735
- await core.nodes("test:str=foo [ -#haha ]")
736
-
737
- splices = await alist(layr.splices(spliceoffs, 10))
738
-
739
- splice = splices[0][1]
740
- self.eq(splice[0], 'tag:del')
741
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
742
- self.eq(splice[1]['tag'], 'haha')
743
- self.eq(splice[1]['valu'], (946684800000, 978307200001))
744
- self.eq(splice[1]['user'], root.iden)
745
- self.nn(splice[1].get('time'))
746
-
747
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
748
-
749
- # Convert a tag:prop:add splice with no oldv
750
- await core.nodes("test:str=foo [ +#rep:risk=50 ]")
751
-
752
- splices = await alist(layr.splices(spliceoffs, 10))
753
-
754
- splice = splices[5][1]
755
- self.eq(splice[0], 'tag:prop:set')
756
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
757
- self.eq(splice[1]['tag'], 'rep')
758
- self.eq(splice[1]['prop'], 'risk')
759
- self.eq(splice[1]['valu'], 50)
760
- self.eq(splice[1]['oldv'], None)
761
- self.eq(splice[1]['user'], root.iden)
762
- self.nn(splice[1].get('time'))
763
-
764
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
765
-
766
- # Convert a tag:prop:add splice with an oldv
767
- await core.nodes("test:str=foo [ +#rep:risk=0 ]")
768
-
769
- splices = await alist(layr.splices(spliceoffs, 10))
770
-
771
- splice = splices[0][1]
772
- self.eq(splice[0], 'tag:prop:set')
773
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
774
- self.eq(splice[1]['tag'], 'rep')
775
- self.eq(splice[1]['prop'], 'risk')
776
- self.eq(splice[1]['valu'], 0)
777
- self.eq(splice[1]['oldv'], 50)
778
- self.eq(splice[1]['user'], root.iden)
779
- self.nn(splice[1].get('time'))
780
-
781
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
782
-
783
- # Convert a tag:prop:del splice
784
- await core.nodes("test:str=foo [ -#rep:risk ]")
785
-
786
- splices = await alist(layr.splices(spliceoffs, 10))
787
-
788
- splice = splices[0][1]
789
- self.eq(splice[0], 'tag:prop:del')
790
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
791
- self.eq(splice[1]['tag'], 'rep')
792
- self.eq(splice[1]['prop'], 'risk')
793
- self.eq(splice[1]['valu'], 0)
794
- self.eq(splice[1]['user'], root.iden)
795
- self.nn(splice[1].get('time'))
796
-
797
- spliceoffs = (splices[-1][0][0] + 1, 0, 0)
798
-
799
- # Nodedata edits don't make splices
800
- nodes = await core.nodes('test:str=foo')
801
- await nodes[0].setData('baz', 'nodedataiscool')
802
-
803
- splices = await alist(layr.splices(spliceoffs, 10))
804
- self.len(0, splices)
805
-
806
- # Make sure nodedata edits have oldv set
807
- nedit = await layr.iterNodeEditLog(spliceoffs[0]).__anext__()
808
- self.eq(nedit[1][0][2][0][1], ('baz', 'nodedataiscool', None))
809
-
810
- spliceoffs = (spliceoffs[0] + 1, 0, 0)
811
-
812
- await nodes[0].setData('baz', 'stillcool')
813
- nedit = await layr.iterNodeEditLog(spliceoffs[0]).__anext__()
814
- self.eq(nedit[1][0][2][0][1], ('baz', 'stillcool', 'nodedataiscool'))
815
-
816
- # Convert a node:del splice
817
- await core.nodes('test:str=foo | delnode')
818
-
819
- splices = await alist(layr.splices(spliceoffs, 10))
820
-
821
- splice = splices[2][1]
822
- self.eq(splice[0], 'node:del')
823
- self.eq(splice[1]['ndef'], ('test:str', 'foo'))
824
- self.eq(splice[1]['user'], root.iden)
825
- self.nn(splice[1].get('time'))
826
-
827
- # Get all the splices
828
- splices = [x async for x in layr.splices()]
829
- self.len(26, splices)
830
-
831
- # Get all but the first splice
832
- await self.agenlen(25, layr.splices(splices[1][0]))
833
-
834
- await self.agenlen(4, layr.splicesBack(splices[3][0]))
835
-
836
- # Make sure we still get two splices when
837
- # offset is not at the beginning of a nodeedit
838
- await self.agenlen(2, layr.splices((1, 0, 200), 2))
839
- await self.agenlen(2, layr.splicesBack((3, 0, -1), 2))
840
-
841
- # Use the layer api to get the splices
842
- url = core.getLocalUrl('*/layer')
843
- async with await s_telepath.openurl(url) as layrprox:
844
- await self.agenlen(26, layrprox.splices())
845
-
846
628
  async def test_layer_stortype_hier(self):
847
629
  stor = s_layer.StorTypeHier(None, None)
848
630
 
@@ -1256,11 +1038,15 @@ class LayerTest(s_t_utils.SynTest):
1256
1038
  nodelist1 = [node.pack() for node in nodelist1]
1257
1039
  self.eq(nodelist0, nodelist1)
1258
1040
 
1259
- layr = core1.view.layers[0]
1041
+ layr = core1.view.layers[0] # type: s_layer.Layer
1260
1042
 
1261
- # Empty the layer to try again
1043
+ ############################################################################
1044
+ # TEST ONLY - Force the layer nexus handler to consume a truncate event.
1045
+ # This is for backwards compatibility for a mirror that consumes a truncate
1046
+ # event.
1047
+ # This can be removed in 3.0.0.
1262
1048
 
1263
- await layr.truncate()
1049
+ await layr._push('layer:truncate')
1264
1050
 
1265
1051
  async with await s_telepath.openurl(url) as layrprox:
1266
1052
 
@@ -1278,7 +1064,7 @@ class LayerTest(s_t_utils.SynTest):
1278
1064
  'time': 0,
1279
1065
  }
1280
1066
 
1281
- await layr.truncate()
1067
+ await layr._push('layer:truncate')
1282
1068
 
1283
1069
  for nodeedits in editlist:
1284
1070
  self.none(await layrprox.storNodeEditsNoLift(nodeedits, meta=meta))
@@ -1300,6 +1086,8 @@ class LayerTest(s_t_utils.SynTest):
1300
1086
  core1.schedCoro(doEdit())
1301
1087
  await asyncio.wait_for(waitForEdit(), timeout=6)
1302
1088
 
1089
+ ############################################################################
1090
+
1303
1091
  async def test_layer_stornodeedits_nonexus(self):
1304
1092
  # test for migration methods that store nodeedits bypassing nexus
1305
1093
 
@@ -1494,9 +1282,9 @@ class LayerTest(s_t_utils.SynTest):
1494
1282
  '''
1495
1283
  await core.nodes('[test:str=foo .seen=(2015, 2016)]')
1496
1284
  layr = core.getLayer(None)
1497
- lbefore = len(await alist(layr.splices()))
1285
+ lbefore = len(await alist(layr.syncNodeEdits2(0, wait=False)))
1498
1286
  await core.nodes('[test:str=foo .seen=(2015, 2016)]')
1499
- lafter = len(await alist(layr.splices()))
1287
+ lafter = len(await alist(layr.syncNodeEdits2(0, wait=False)))
1500
1288
  self.eq(lbefore, lafter)
1501
1289
 
1502
1290
  async def test_layer_del_then_lift(self):
@@ -1561,9 +1349,6 @@ class LayerTest(s_t_utils.SynTest):
1561
1349
  readlayr = core.getLayer(readlayrinfo.get('iden'))
1562
1350
  self.true(readlayr.readonly)
1563
1351
 
1564
- with self.raises(s_exc.IsReadOnly):
1565
- await readlayr.truncate()
1566
-
1567
1352
  async def test_layer_ro(self):
1568
1353
  with self.getTestDir() as dirn:
1569
1354
  async with self.getTestCore(dirn=dirn) as core:
@@ -1430,6 +1430,28 @@ class LmdbSlabTest(s_t_utils.SynTest):
1430
1430
  self.len(6, items)
1431
1431
  self.eq(items, ['bot', 'timewarp', 'foo', 'bar', 'baz', 'eot'])
1432
1432
 
1433
+ async def test_lmdbslab_count(self):
1434
+
1435
+ with self.getTestDir() as dirn:
1436
+
1437
+ path = os.path.join(dirn, 'test.lmdb')
1438
+ async with await s_lmdbslab.Slab.anit(path) as slab:
1439
+
1440
+ testdb = slab.initdb('test')
1441
+ dupsdb = slab.initdb('dups', dupsort=True)
1442
+
1443
+ self.eq(0, slab.count(b'newp', db=testdb))
1444
+ self.eq(0, slab.count(b'newp', db=dupsdb))
1445
+
1446
+ slab.put(b'foo', b'bar', db=testdb)
1447
+
1448
+ self.eq(1, slab.count(b'foo', db=testdb))
1449
+
1450
+ slab.put(b'foo', b'bar', db=dupsdb)
1451
+ slab.put(b'foo', b'baz', db=dupsdb)
1452
+ slab.put(b'foo', b'faz', db=dupsdb)
1453
+
1454
+ self.eq(3, slab.count(b'foo', db=dupsdb))
1433
1455
 
1434
1456
  class LmdbSlabMemLockTest(s_t_utils.SynTest):
1435
1457
 
@@ -625,3 +625,12 @@ class SnapTest(s_t_utils.SynTest):
625
625
  self.len(1, await core.nodes('test:ro=foo [ :readable = haha ]'))
626
626
  with self.raises(s_exc.ReadOnlyProp):
627
627
  await core.nodes('test:ro=foo [ :readable=newp ]')
628
+
629
+ async def test_snap_subs_depth(self):
630
+
631
+ async with self.getTestCore() as core:
632
+ fqdn = '.'.join(['x' for x in range(300)]) + '.foo.com'
633
+ q = f'[ inet:fqdn="{fqdn}"]'
634
+ nodes = await core.nodes(q)
635
+ self.len(1, nodes)
636
+ self.eq(nodes[0].get('zone'), 'foo.com')