synapse 2.186.0__py311-none-any.whl → 2.187.1__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

@@ -1,9 +1,13 @@
1
1
  import datetime
2
+ import textwrap
3
+
4
+ from unittest import mock
2
5
 
3
6
  import synapse.exc as s_exc
4
7
  import synapse.common as s_common
5
8
 
6
9
  import synapse.lib.chop as s_chop
10
+ import synapse.lib.spooled as s_spooled
7
11
  import synapse.lib.modelrev as s_modelrev
8
12
 
9
13
  import synapse.tests.utils as s_tests
@@ -126,7 +130,7 @@ class ModelRevTest(s_tests.SynTest):
126
130
  url3 = "https://alpha.com/woot"
127
131
 
128
132
  # created via: f'[it:sec:cve=CVE-2013-9999 :desc="some words" :references=({url0}, {url1}, {url2}, {url3})]'
129
- nodes = await core.nodes(f'it:sec:cve=CVE-2013-9999')
133
+ nodes = await core.nodes('it:sec:cve=CVE-2013-9999')
130
134
  self.eq(nodes[0].ndef[1], 'cve-2013-9999')
131
135
  self.eq(nodes[0].get('desc'), 'some words')
132
136
  self.eq(nodes[0].get('references'), (url3, url1, url0, url2))
@@ -569,47 +573,106 @@ class ModelRevTest(s_tests.SynTest):
569
573
  self.len(1, nodes)
570
574
  self.eq('Foo', nodes[0].get('id'))
571
575
 
572
- async def test_modelrev_cpe(self):
573
- # Skip this test for now since 0_2_28 migration was disabled. This test will be moved/updated in a later PR.
574
- self.skip('Model 0.2.28 disabled')
576
+ async def test_modelrev_0_2_29(self):
577
+ async with self.getRegrCore('model-0.2.29') as core:
578
+ self.len(2, await core.nodes('ou:industry:type:taxonomy'))
579
+
580
+ async def test_modelrev_0_2_30(self):
581
+ async with self.getRegrCore('model-0.2.30') as core:
582
+ q = '''
583
+ inet:ipv4=192.0.0.0 inet:ipv4=192.0.0.8 inet:ipv4=192.0.0.9 inet:ipv4=192.0.0.10 inet:ipv4=192.0.0.255
584
+ '''
585
+ nodes = await core.nodes(q)
586
+ typz = [node.get('type') for node in nodes]
587
+ self.eq(typz, ['private', 'private', 'unicast', 'unicast', 'private'])
588
+
589
+ q = '''
590
+ inet:ipv6="64:ff9b:1::" inet:ipv6="64:ff9b:1::1" inet:ipv6="64:ff9b:1::ffff" inet:ipv6="64:ff9b:1::ffff:1"
591
+ '''
592
+ nodes = await core.nodes(q)
593
+ typz = [node.get('type') for node in nodes]
594
+ self.eq(typz, ['private', 'private', 'private', 'private'])
595
+
596
+ q = '''
597
+ inet:ipv6="2002::" inet:ipv6="2002::1" inet:ipv6="2002::fffe" inet:ipv6="2002::ffff"
598
+ '''
599
+ nodes = await core.nodes(q)
600
+ typz = [node.get('type') for node in nodes]
601
+ self.eq(typz, ['private', 'private', 'private', 'private'])
602
+
603
+ q = 'inet:ipv6="2001:1::1/128" inet:ipv6="2001:1::2/128"'
604
+ nodes = await core.nodes(q)
605
+ typz = [node.get('type') for node in nodes]
606
+ self.eq(typz, ['unicast', 'unicast'])
607
+
608
+ q = 'inet:ipv6="2001:3::" inet:ipv6="2001:3::1" inet:ipv6="2001:3::ffff"'
609
+ nodes = await core.nodes(q)
610
+ typz = [node.get('type') for node in nodes]
611
+ self.eq(typz, ['unicast', 'unicast', 'unicast'])
612
+
613
+ q = 'inet:ipv6="2001:4:112::" inet:ipv6="2001:4:112::1" inet:ipv6="2001:4:112::ffff"'
614
+ nodes = await core.nodes(q)
615
+ typz = [node.get('type') for node in nodes]
616
+ self.eq(typz, ['unicast', 'unicast', 'unicast'])
617
+
618
+ q = 'inet:ipv6="2001:20::" inet:ipv6="2001:20::1" inet:ipv6="2001:20::ffff"'
619
+ nodes = await core.nodes(q)
620
+ typz = [node.get('type') for node in nodes]
621
+ self.eq(typz, ['unicast', 'unicast', 'unicast'])
622
+
623
+ q = 'inet:ipv6="2001:30::" inet:ipv6="2001:30::1" inet:ipv6="2001:30::ffff"'
624
+ nodes = await core.nodes(q)
625
+ typz = [node.get('type') for node in nodes]
626
+ self.eq(typz, ['unicast', 'unicast', 'unicast'])
627
+
628
+ async def test_modelrev_0_2_31(self):
575
629
 
576
- async with self.getRegrCore('model-0.2.28', maxvers=(0, 2, 24)) as core:
630
+ self.maxDiff = None
631
+
632
+ async with self.getRegrCore('model-cpe-migration', maxvers=(0, 2, 24)) as core:
577
633
  # Do some pre-migration validation of the cortex. It's still a
578
634
  # little weird in here because the CPE types have been updated so
579
635
  # some lifting/pivoting won't work right.
580
636
 
581
- views = await core.callStorm('return($lib.view.list(deporder=$lib.true))')
582
- self.len(3, views)
637
+ # There should be nothing in the default view
638
+ nodes = await core.nodes('.created')
639
+ self.len(0, nodes)
640
+
641
+ views = {view.info.get('name'): view for view in core.listViews()}
642
+ self.len(5, views)
583
643
 
584
- fork00 = views[1].get('iden')
644
+ fork00 = views.get('fork00').iden
585
645
  infork00 = {'view': fork00}
586
646
 
587
- nodes = await core.nodes('it:sec:cpe')
588
- self.len(11, nodes)
647
+ fork01 = views.get('fork01').iden
648
+ infork01 = {'view': fork01}
649
+
650
+ nodes = await core.nodes('it:sec:cpe', opts=infork00)
651
+ self.len(12, nodes)
589
652
  for node in nodes:
590
653
  self.isin('test.cpe', node.tags)
591
654
  data = await s_tests.alist(node.iterData())
592
655
  self.eq([k[0] for k in data], ('cpe22', 'cpe23'))
593
656
 
594
- nodes = await core.nodes('it:sec:cpe -(refs)> risk:vuln | uniq')
657
+ nodes = await core.nodes('it:sec:cpe -(refs)> risk:vuln | uniq', opts=infork00)
595
658
  self.len(1, nodes)
596
659
  self.eq(nodes[0].ndef, ('risk:vuln', s_common.guid(('risk', 'vuln'))))
597
660
 
598
- nodes = await core.nodes('risk:vulnerable')
599
- self.len(11, nodes)
661
+ nodes = await core.nodes('risk:vulnerable', opts=infork00)
662
+ self.len(12, nodes)
600
663
  for node in nodes:
601
664
  self.nn(node.get('node'))
602
665
 
603
- nodes = await core.nodes(r'it:sec:cpe:vendor="d\-link"')
666
+ nodes = await core.nodes(r'it:sec:cpe:vendor="d\-link"', opts=infork00)
604
667
  self.len(1, nodes)
605
668
 
606
- nodes = await core.nodes('it:prod:soft', opts=infork00)
669
+ nodes = await core.nodes('it:prod:soft', opts=infork01)
607
670
  self.len(4, nodes)
608
671
  for node in nodes:
609
672
  self.isin('test.prod', node.tags)
610
673
  self.nn(node.get('cpe'))
611
674
 
612
- nodes = await core.nodes('inet:flow', opts=infork00)
675
+ nodes = await core.nodes('inet:flow', opts=infork01)
613
676
  self.len(4, nodes)
614
677
  for node in nodes:
615
678
  self.isin('test.flow', node.tags)
@@ -620,35 +683,56 @@ class ModelRevTest(s_tests.SynTest):
620
683
  (srcs is not None and len(srcs) == 2)
621
684
  ))
622
685
 
623
- nodes = await core.nodes('_ext:model:form', opts=infork00)
686
+ nodes = await core.nodes('_ext:model:form', opts=infork01)
624
687
  self.len(4, nodes)
625
688
  for node in nodes:
626
689
  self.isin('test.ext', node.tags)
627
690
  self.nn(node.get('cpe'))
628
691
 
629
- nodes = await core.nodes('meta:source:name="cpe.22.invalid" -(seen)> it:sec:cpe', opts=infork00)
630
- self.len(5, nodes)
631
-
632
- nodes = await core.nodes('meta:source:name="cpe.23.invalid" -(seen)> it:sec:cpe', opts=infork00)
692
+ nodes = await core.nodes('meta:source:name="cpe.22.invalid" -(seen)> it:sec:cpe', opts=infork01)
633
693
  self.len(6, nodes)
634
694
 
635
- nodes = await core.nodes('meta:source:name="cpe.22.invalid" -> meta:seen', opts=infork00)
636
- self.len(5, nodes)
695
+ nodes = await core.nodes('meta:source:name="cpe.23.invalid" -(seen)> it:sec:cpe', opts=infork01)
696
+ self.len(7, nodes)
637
697
 
638
- nodes = await core.nodes('meta:source:name="cpe.23.invalid" -> meta:seen', opts=infork00)
698
+ nodes = await core.nodes('meta:source:name="cpe.22.invalid" -> meta:seen', opts=infork01)
639
699
  self.len(6, nodes)
640
700
 
641
- nodes = await core.nodes('it:sec:vuln:scan:result', opts=infork00)
642
- self.len(11, nodes)
701
+ nodes = await core.nodes('meta:source:name="cpe.23.invalid" -> meta:seen', opts=infork01)
702
+ self.len(7, nodes)
703
+
704
+ nodes = await core.nodes('it:sec:vuln:scan:result', opts=infork01)
705
+ self.len(13, nodes)
706
+
707
+ # Do some error checking before the queue is created
708
+ q = '''
709
+ for $entry in $lib.model.migration.s.model_0_2_31.listNodes() {
710
+ $lib.print(`ENTRY: {$entry}`)
711
+ }
712
+ '''
713
+ msgs = await core.stormlist(q)
714
+ self.stormIsInPrint('Queue model_0_2_31:nodes not found, no nodes to list.', msgs)
715
+
716
+ msgs = await core.stormlist('$lib.model.migration.s.model_0_2_31.printNode((0))')
717
+ self.stormIsInPrint('Queue model_0_2_31:nodes not found, no nodes to print.', msgs)
643
718
 
644
- async with self.getRegrCore('model-0.2.28') as core:
719
+ msgs = await core.stormlist('$lib.model.migration.s.model_0_2_31.repairNode((0), newp)')
720
+ self.stormIsInPrint('Queue model_0_2_31:nodes not found, no nodes to repair.', msgs)
645
721
 
646
- views = await core.callStorm('return($lib.view.list(deporder=$lib.true))')
647
- self.len(3, views)
722
+ async with self.getRegrCore('model-cpe-migration') as core:
648
723
 
649
- fork00 = views[1].get('iden')
724
+ views = {view.info.get('name'): view for view in core.listViews()}
725
+ self.len(5, views)
726
+
727
+ fork00 = views.get('fork00').iden
650
728
  infork00 = {'view': fork00}
651
729
 
730
+ fork01 = views.get('fork01').iden
731
+ infork01 = {'view': fork01}
732
+
733
+ fork03 = views.get('fork03').iden
734
+ infork03 = {'view': fork03}
735
+
652
736
  # Calculate some timestamps
653
737
  start = datetime.datetime(year=2020, month=1, day=1, tzinfo=datetime.timezone.utc)
654
738
  end = datetime.datetime(year=2021, month=1, day=1, tzinfo=datetime.timezone.utc)
@@ -656,9 +740,9 @@ class ModelRevTest(s_tests.SynTest):
656
740
  start = int(start.timestamp() * 1000)
657
741
  end = int(end.timestamp() * 1000)
658
742
 
659
- # We started with 11 CPE nodes and two got removed
660
- nodes = await core.nodes('it:sec:cpe')
661
- self.len(9, nodes)
743
+ # We started with 12 CPE nodes and one got removed
744
+ nodes = await core.nodes('it:sec:cpe', opts=infork00)
745
+ self.len(11, nodes)
662
746
  for node in nodes:
663
747
  self.isin('test.cpe', node.tags)
664
748
  data = await s_tests.alist(node.iterData())
@@ -670,33 +754,37 @@ class ModelRevTest(s_tests.SynTest):
670
754
 
671
755
  self.eq((start, end), seen)
672
756
 
673
- nodes = await core.nodes('it:sec:cpe#test.cpe.22invalid +#test.cpe.23invalid')
674
- self.len(0, nodes)
757
+ nodes = await core.nodes('it:sec:cpe', opts=infork03)
758
+ self.len(1, nodes)
759
+ self.eq(nodes[0].repr(), r'cpe:2.3:a:\@ianwalter:merge:*:*:*:*:*:*:*:*')
675
760
 
676
- nodes = await core.nodes('it:sec:cpe -(refs)> risk:vuln')
677
- self.len(9, nodes)
761
+ nodes = await core.nodes('it:sec:cpe#test.cpe.22invalid +#test.cpe.23invalid', opts=infork00)
762
+ self.len(2, nodes)
678
763
 
679
- nodes = await core.nodes('risk:vulnerable')
764
+ nodes = await core.nodes('it:sec:cpe -(refs)> risk:vuln', opts=infork00)
680
765
  self.len(11, nodes)
681
766
 
682
- nodes = await core.nodes('risk:vulnerable:node')
683
- self.len(9, nodes)
767
+ nodes = await core.nodes('risk:vulnerable', opts=infork00)
768
+ self.len(12, nodes)
684
769
 
685
- nodes = await core.nodes('risk:vulnerable -> it:sec:cpe')
686
- self.len(9, nodes)
770
+ nodes = await core.nodes('risk:vulnerable:node', opts=infork00)
771
+ self.len(11, nodes)
687
772
 
688
- nodes = await core.nodes('risk:vulnerable -:node')
689
- self.len(2, nodes)
773
+ nodes = await core.nodes('risk:vulnerable -> it:sec:cpe', opts=infork00)
774
+ self.len(11, nodes)
690
775
 
691
- nodes = await core.nodes('it:prod:soft', opts=infork00)
776
+ nodes = await core.nodes('risk:vulnerable -:node', opts=infork00)
777
+ self.len(1, nodes)
778
+
779
+ nodes = await core.nodes('it:prod:soft', opts=infork01)
692
780
  self.len(4, nodes)
693
781
  for node in nodes:
694
782
  self.isin('test.prod', node.tags)
695
783
 
696
- nodes = await core.nodes('it:prod:soft:cpe', opts=infork00)
784
+ nodes = await core.nodes('it:prod:soft:cpe', opts=infork01)
697
785
  self.len(3, nodes)
698
786
 
699
- nodes = await core.nodes('it:prod:soft -> it:sec:cpe', opts=infork00)
787
+ nodes = await core.nodes('it:prod:soft -> it:sec:cpe', opts=infork01)
700
788
  self.len(3, nodes)
701
789
  ndefs = [k.ndef for k in nodes]
702
790
  self.sorteq(ndefs, (
@@ -705,24 +793,28 @@ class ModelRevTest(s_tests.SynTest):
705
793
  ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas326_firmware:5.21\\(aazf.14\\)c0:*:*:*:*:*:*:*'),
706
794
  ))
707
795
 
708
- nodes = await core.nodes('it:prod:soft -:cpe', opts=infork00)
796
+ nodes = await core.nodes('it:prod:soft -:cpe', opts=infork01)
709
797
  self.len(1, nodes)
710
798
  self.eq(nodes[0].get('name'), '22i-23i')
711
799
 
712
- nodes = await core.nodes('inet:flow', opts=infork00)
800
+ nodes = await core.nodes('inet:flow', opts=infork01)
713
801
  self.len(4, nodes)
714
802
 
715
- nodes = await core.nodes('inet:flow +(:src:cpes or :dst:cpes)', opts=infork00)
716
- self.len(3, nodes)
803
+ nodes = await core.nodes('inet:flow +(:src:cpes or :dst:cpes)', opts=infork01)
804
+ self.len(4, nodes)
805
+
806
+ nodes = await core.nodes('inet:flow -(:src:cpes or :dst:cpes)', opts=infork01)
807
+ self.len(0, nodes)
717
808
 
718
- nodes = await core.nodes('inet:flow -(:src:cpes or :dst:cpes)', opts=infork00)
809
+ nodes = await core.nodes('inet:flow=(flow, 22i, 23i)', opts=infork01)
719
810
  self.len(1, nodes)
720
- self.eq(nodes[0].repr(), s_common.guid(('flow', '22i', '23i')))
811
+ self.none(nodes[0].get('dst:cpes'))
721
812
 
722
- nodes = await core.nodes('inet:flow -> it:sec:cpe', opts=infork00)
723
- self.len(6, nodes)
813
+ nodes = await core.nodes('inet:flow -> it:sec:cpe', opts=infork01)
814
+ self.len(7, nodes)
724
815
  ndefs = [k.ndef for k in nodes]
725
816
  self.sorteq(ndefs, (
817
+ ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh_server:7.4:*:*:*:*:*:*:*'),
726
818
  ('it:sec:cpe', 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*'),
727
819
  ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas326_firmware:5.21\\(aazf.14\\)c0:*:*:*:*:*:*:*'),
728
820
  ('it:sec:cpe', 'cpe:2.3:a:01generator:pireospay:-:*:*:*:*:prestashop:*:*'),
@@ -731,16 +823,16 @@ class ModelRevTest(s_tests.SynTest):
731
823
  ('it:sec:cpe', 'cpe:2.3:a:abinitio:control\\>center:-:*:*:*:*:*:*:*'),
732
824
  ))
733
825
 
734
- nodes = await core.nodes('_ext:model:form', opts=infork00)
826
+ nodes = await core.nodes('_ext:model:form', opts=infork01)
735
827
  self.len(4, nodes)
736
828
 
737
- nodes = await core.nodes('_ext:model:form:cpe', opts=infork00)
829
+ nodes = await core.nodes('_ext:model:form:cpe', opts=infork01)
738
830
  self.len(3, nodes)
739
831
 
740
- nodes = await core.nodes('_ext:model:form -:cpe', opts=infork00)
832
+ nodes = await core.nodes('_ext:model:form -:cpe', opts=infork01)
741
833
  self.len(1, nodes)
742
834
 
743
- nodes = await core.nodes('_ext:model:form -> it:sec:cpe', opts=infork00)
835
+ nodes = await core.nodes('_ext:model:form -> it:sec:cpe', opts=infork01)
744
836
  self.len(3, nodes)
745
837
  ndefs = [k.ndef for k in nodes]
746
838
  self.sorteq(ndefs, (
@@ -749,10 +841,10 @@ class ModelRevTest(s_tests.SynTest):
749
841
  ('it:sec:cpe', r'cpe:2.3:a:1c:1c\:enterprise:-:*:*:*:*:*:*:*'),
750
842
  ))
751
843
 
752
- nodes = await core.nodes('meta:seen', opts=infork00)
844
+ nodes = await core.nodes('meta:seen', opts=infork01)
753
845
  self.len(3, nodes)
754
846
 
755
- nodes = await core.nodes('meta:seen -> it:sec:cpe', opts=infork00)
847
+ nodes = await core.nodes('meta:seen -> it:sec:cpe', opts=infork01)
756
848
  self.len(3, nodes)
757
849
  ndefs = [k.ndef for k in nodes]
758
850
  self.sorteq(ndefs, (
@@ -761,7 +853,7 @@ class ModelRevTest(s_tests.SynTest):
761
853
  ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas542_firmware:5.21\\%28aazf.15\\%29co:*:*:*:*:*:*:*'),
762
854
  ))
763
855
 
764
- nodes = await core.nodes('it:sec:cpe -> meta:seen -> it:sec:vuln:scan:result', opts=infork00)
856
+ nodes = await core.nodes('it:sec:cpe -> meta:seen -> it:sec:vuln:scan:result', opts=infork01)
765
857
  self.len(3, nodes)
766
858
  ndefs = [k.ndef for k in nodes]
767
859
  self.sorteq(ndefs, (
@@ -771,84 +863,111 @@ class ModelRevTest(s_tests.SynTest):
771
863
  ))
772
864
 
773
865
  # Check that we correctly copied over the edges
774
- nodes = await core.nodes('risk:vuln <(refs)- it:sec:cpe')
775
- self.len(9, nodes)
866
+ nodes = await core.nodes('risk:vuln <(refs)- it:sec:cpe', opts=infork00)
867
+ self.len(11, nodes)
776
868
 
777
869
  # Check that we correctly copied over the tags
778
- nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:o:zyxel:nas326_firmware:5.21\(aazf.14\)c0:*:*:*:*:*:*:*"')
870
+ nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:o:zyxel:nas326_firmware:5.21\(aazf.14\)c0:*:*:*:*:*:*:*"', opts=infork00)
779
871
  self.len(1, nodes)
780
872
  self.isin('test.cpe.22valid', nodes[0].tags)
781
873
  self.isin('test.cpe.23invalid', nodes[0].tags)
782
874
 
783
- nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*"')
875
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*"', opts=infork00)
784
876
  self.len(1, nodes)
785
877
  self.isin('test.cpe.22valid', nodes[0].tags)
786
878
  self.isin('test.cpe.23invalid', nodes[0].tags)
787
879
 
788
- nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:a:acurax:under_construction_\/_maintenance_mode:-:*:*:*:*:wordpress:*:*"')
880
+ nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:a:acurax:under_construction_\/_maintenance_mode:-:*:*:*:*:wordpress:*:*"', opts=infork00)
789
881
  self.len(1, nodes)
790
882
  self.isin('test.cpe.22valid', nodes[0].tags)
791
883
  self.isin('test.cpe.23invalid', nodes[0].tags)
792
884
 
793
- nodes = await core.nodes('it:sec:cpe="cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*"')
885
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*"', opts=infork00)
794
886
  self.len(1, nodes)
795
887
  self.isin('test.cpe.22valid', nodes[0].tags)
796
888
  self.isin('test.cpe.23invalid', nodes[0].tags)
797
889
 
890
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:openbsd:openssh_server:7.4:*:*:*:*:*:*:*"', opts=infork00)
891
+ self.len(1, nodes)
892
+ self.isin('test.cpe.22invalid', nodes[0].tags)
893
+ self.isin('test.cpe.23invalid', nodes[0].tags)
894
+
798
895
  # Check that we correctly copied over the node data
799
- nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:o:zyxel:nas326_firmware:5.21\(aazf.14\)c0:*:*:*:*:*:*:*"')
896
+ nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:o:zyxel:nas326_firmware:5.21\(aazf.14\)c0:*:*:*:*:*:*:*"', opts=infork00)
800
897
  self.len(1, nodes)
801
898
  data = await s_tests.alist(nodes[0].iterData())
802
899
  self.sorteq(data, (('cpe23', 'invalid'), ('cpe22', 'valid')))
803
900
 
804
- nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*"')
901
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*"', opts=infork00)
805
902
  self.len(1, nodes)
806
903
  data = await s_tests.alist(nodes[0].iterData())
807
904
  self.sorteq(data, (('cpe23', 'invalid'), ('cpe22', 'valid')))
808
905
 
809
- nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:a:acurax:under_construction_\/_maintenance_mode:-:*:*:*:*:wordpress:*:*"')
906
+ nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:a:acurax:under_construction_\/_maintenance_mode:-:*:*:*:*:wordpress:*:*"', opts=infork00)
810
907
  self.len(1, nodes)
811
908
  data = await s_tests.alist(nodes[0].iterData())
812
909
  self.sorteq(data, (('cpe23', 'invalid'), ('cpe22', 'valid')))
813
910
 
814
- nodes = await core.nodes('it:sec:cpe="cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*"')
911
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*"', opts=infork00)
815
912
  self.len(1, nodes)
816
913
  data = await s_tests.alist(nodes[0].iterData())
817
914
  self.sorteq(data, (('cpe23', 'invalid'), ('cpe22', 'valid')))
818
915
 
916
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:openbsd:openssh_server:7.4:*:*:*:*:*:*:*"', opts=infork00)
917
+ self.len(1, nodes)
918
+ data = await s_tests.alist(nodes[0].iterData())
919
+ self.sorteq(data, (('cpe23', 'invalid'), ('cpe22', 'invalid')))
920
+
819
921
  # Check that we correctly copied over the extended props
820
- nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:o:zyxel:nas326_firmware:5.21\(aazf.14\)c0:*:*:*:*:*:*:*"')
922
+ nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:o:zyxel:nas326_firmware:5.21\(aazf.14\)c0:*:*:*:*:*:*:*"', opts=infork00)
821
923
  self.len(1, nodes)
822
924
  self.true(nodes[0].get('_cpe22valid'))
823
925
  self.false(nodes[0].get('_cpe23valid'))
824
926
 
825
- nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*"')
927
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:10web:social_feed_for_instagram:1.0.0:*:*:*:premium:wordpress:*:*"', opts=infork00)
826
928
  self.len(1, nodes)
827
929
  self.true(nodes[0].get('_cpe22valid'))
828
930
  self.false(nodes[0].get('_cpe23valid'))
829
931
 
830
- nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:a:acurax:under_construction_\/_maintenance_mode:-:*:*:*:*:wordpress:*:*"')
932
+ nodes = await core.nodes(r'it:sec:cpe="cpe:2.3:a:acurax:under_construction_\/_maintenance_mode:-:*:*:*:*:wordpress:*:*"', opts=infork00)
831
933
  self.len(1, nodes)
832
934
  self.true(nodes[0].get('_cpe22valid'))
833
935
  self.false(nodes[0].get('_cpe23valid'))
834
936
 
835
- nodes = await core.nodes('it:sec:cpe="cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*"')
937
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:h:d-link:dir-850l:*:*:*:*:*:*:*:*"', opts=infork00)
836
938
  self.len(1, nodes)
837
939
  self.true(nodes[0].get('_cpe22valid'))
838
940
  self.false(nodes[0].get('_cpe23valid'))
839
941
 
840
- async with self.getRegrCore('model-0.2.28') as core:
942
+ nodes = await core.nodes('it:sec:cpe="cpe:2.3:a:openbsd:openssh_server:7.4:*:*:*:*:*:*:*"', opts=infork00)
943
+ self.len(1, nodes)
944
+ self.false(nodes[0].get('_cpe22valid'))
945
+ self.false(nodes[0].get('_cpe23valid'))
946
+
947
+ # There should be nothing in the default view
948
+ nodes = await core.nodes('.created')
949
+ self.len(0, nodes)
950
+
951
+ async with self.getRegrCore('model-cpe-migration') as core:
841
952
 
842
- views = await core.callStorm('return($lib.view.list(deporder=$lib.true))')
843
- self.len(3, views)
953
+ views = {view.info.get('name'): view for view in core.listViews()}
954
+ self.len(5, views)
844
955
 
845
- fork00 = views[1].get('iden') # forked view
846
- forklayr = views[1].get('layers')[0].get('iden')
956
+ fork00 = views.get('fork00').iden
957
+ fork00layr = views.get('fork00').layers[0].iden
958
+ infork00 = {'view': fork00}
847
959
 
848
- view00 = views[0].get('iden') # default view
849
- viewlayr = views[0].get('layers')[0].get('iden')
960
+ fork01 = views.get('fork01').iden
961
+ fork01layr = views.get('fork01').layers[0].iden
962
+ infork01 = {'view': fork01}
850
963
 
851
- opts = {'view': fork00}
964
+ fork02 = views.get('fork02').iden # forked view
965
+
966
+ fork03 = views.get('fork03').iden
967
+ fork03layr = views.get('fork03').layers[0].iden
968
+ infork03 = {'view': fork03}
969
+
970
+ opts = {'view': fork01}
852
971
 
853
972
  nodes = await core.nodes('meta:source:name="cpe.22.invalid"', opts=opts)
854
973
  self.len(1, nodes)
@@ -864,12 +983,15 @@ class ModelRevTest(s_tests.SynTest):
864
983
  source23 = source01.ndef[1]
865
984
  source23iden = source01.iden()
866
985
 
986
+ riskvuln = s_common.ehex(s_common.buid(('risk:vuln', s_common.guid(('risk', 'vuln')))))
987
+
867
988
  invcpe00 = 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0::~~premium~wordpress~~:*:*:*:*:*'
868
989
  invcpe01 = 'cpe:2.3:a:acurax:under_construction_%2f_maintenance_mode:-::~~~wordpress~~:*:*:*:*:*'
869
990
  invcpe02 = 'cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*'
870
991
  invcpe03 = 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*'
871
992
  invcpe04 = 'cpe:2.3:h:d\\-link:dir\\-850l:*:*:*:*:*:*:*:*'
872
993
  invcpe05 = 'cpe:2.3:o:zyxel:nas326_firmware:5.21%28aazf.14%29c0:*:*:*:*:*:*:*'
994
+ invcpe06 = 'cpe:2.3:a:%40ianwalter:merge:*:*:*:*:*:*:*:*'
873
995
 
874
996
  metaseen00 = s_common.ehex(s_common.buid(('meta:seen', (source23, ('it:sec:cpe', invcpe00)))))
875
997
  metaseen01 = s_common.ehex(s_common.buid(('meta:seen', (source23, ('it:sec:cpe', invcpe01)))))
@@ -879,344 +1001,761 @@ class ModelRevTest(s_tests.SynTest):
879
1001
  metaseen05 = s_common.ehex(s_common.buid(('meta:seen', (source22, ('it:sec:cpe', invcpe03)))))
880
1002
  metaseen06 = s_common.ehex(s_common.buid(('meta:seen', (source23, ('it:sec:cpe', invcpe04)))))
881
1003
  metaseen07 = s_common.ehex(s_common.buid(('meta:seen', (source23, ('it:sec:cpe', invcpe05)))))
1004
+ metaseen08 = s_common.ehex(s_common.buid(('meta:seen', (source22, ('it:sec:cpe', invcpe06)))))
1005
+ metaseen09 = s_common.ehex(s_common.buid(('meta:seen', (source23, ('it:sec:cpe', invcpe06)))))
882
1006
 
883
- badcpe00 = s_common.ehex(s_common.buid(('it:sec:cpe', invcpe02)))
884
- badcpe01 = s_common.ehex(s_common.buid(('it:sec:cpe', invcpe03)))
1007
+ badcpe00 = s_common.ehex(s_common.buid(('it:sec:cpe', invcpe03)))
885
1008
 
886
1009
  '''
887
- There are two CPEs that we couldn't migrate. They should be fully
888
- represented in the following three queues for potentially being
889
- rebuilt later.
1010
+ There is one CPE that we couldn't migrate. It should be fully represented in the following queues for
1011
+ potentially being rebuilt later.
890
1012
 
891
- badcpe00: it:sec:cpe="cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*"
892
- badcpe01: it:sec:cpe="cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*"
1013
+ badcpe00: it:sec:cpe="cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*"
893
1014
  '''
894
1015
 
895
1016
  queues = await core.callStorm('return($lib.queue.list())')
896
1017
  [q.pop('meta') for q in queues]
897
- self.len(3, queues)
1018
+ self.len(1, queues)
898
1019
  self.eq(queues, (
899
- {'name': 'model_0_2_28:nodes', 'size': 10, 'offs': 10},
900
- {'name': 'model_0_2_28:nodes:refs', 'size': 14, 'offs': 14},
901
- {'name': 'model_0_2_28:nodes:edges', 'size': 4, 'offs': 4},
1020
+ {'name': 'model_0_2_31:nodes', 'size': 11, 'offs': 11},
902
1021
  ))
903
1022
 
904
1023
  q = '''
905
1024
  $ret = ([])
906
- $q = $lib.queue.get('model_0_2_28:nodes')
1025
+ $q = $lib.queue.get('model_0_2_31:nodes')
907
1026
  for $ii in $lib.range(($q.size())) {
908
- $ret.append($q.get($ii, cull=(false), wait=(false)))
1027
+ ($offs, $item) = $q.get($ii, cull=(false), wait=(false))
1028
+ $ret.append($item)
909
1029
  }
910
1030
  fini { return($ret) }
911
1031
  '''
912
1032
  nodesq = await core.callStorm(q)
913
1033
  for item in nodesq:
914
- self.nn(item[1]['props'].pop('.created'))
1034
+ if (sources := item.get('sources')):
1035
+ item['sources'] = tuple(sorted(sources))
915
1036
 
916
- self.eq(nodesq, [
917
- (0,
918
- {'data': (),
919
- 'form': 'meta:seen',
920
- 'iden': metaseen00,
921
- 'layer': forklayr,
922
- 'props': {'node': ('it:sec:cpe', invcpe00),
923
- 'source': source23},
924
- 'repr': (source23, ('it:sec:cpe', invcpe00)),
1037
+ if (layers := item.get('layers')):
1038
+ item['layers'] = tuple(sorted(layers))
1039
+
1040
+ self.len(11, nodesq)
1041
+
1042
+ expected = [
1043
+ {'formname': 'meta:seen',
1044
+ 'iden': metaseen08,
1045
+ 'layers': (fork01layr,),
1046
+ 'formvalu': (source22, ('it:sec:cpe', invcpe06)),
925
1047
  'sources': (),
926
- 'tags': (),
927
- 'view': fork00}),
928
- (1,
929
- {'data': (),
930
- 'form': 'meta:seen',
931
- 'iden': metaseen01,
932
- 'layer': forklayr,
933
- 'props': {'node': ('it:sec:cpe', invcpe01),
934
- 'source': source23},
935
- 'repr': (source23, ('it:sec:cpe', invcpe01)),
1048
+ 'sodes': {
1049
+ fork01layr: {
1050
+ 'form': 'meta:seen',
1051
+ 'valu': ((source22, ('it:sec:cpe', invcpe06)), 13),
1052
+ },
1053
+ },
1054
+ 'n1edges': {},
1055
+ 'n2edges': {},
1056
+ 'nodedata': {},
1057
+ 'refs': {
1058
+ fork01layr: (
1059
+ ('5fbce86c228ebf052bebca0bebbadbf3ae92a7afbd35f35996a275e6688ad88e',
1060
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1061
+ ),
1062
+ },
1063
+ },
1064
+ {'formname': 'meta:seen',
1065
+ 'iden': metaseen09,
1066
+ 'layers': (fork01layr,),
1067
+ 'formvalu': (source23, ('it:sec:cpe', invcpe06)),
936
1068
  'sources': (),
937
- 'tags': (),
938
- 'view': fork00}),
939
- (2,
940
- {'data': (),
941
- 'form': 'meta:seen',
1069
+ 'sodes': {
1070
+ fork01layr: {
1071
+ 'form': 'meta:seen',
1072
+ 'valu': ((source23, ('it:sec:cpe', invcpe06)), 13),
1073
+ },
1074
+ },
1075
+ 'n1edges': {},
1076
+ 'n2edges': {},
1077
+ 'nodedata': {},
1078
+ 'refs': {
1079
+ fork01layr: (
1080
+ ('52d48d748a795329651e62f89c22a1f24e3560f1858aec2c5eba304e711c0bf5',
1081
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1082
+ ),
1083
+ },
1084
+ },
1085
+ {'formname': 'meta:seen',
942
1086
  'iden': metaseen02,
943
- 'layer': forklayr,
944
- 'props': {'node': ('it:sec:cpe', invcpe02),
945
- 'source': source22},
946
- 'repr': (source22, ('it:sec:cpe', invcpe02)),
1087
+ 'layers': (fork01layr,),
1088
+ 'formvalu': (source22, ('it:sec:cpe', invcpe02)),
947
1089
  'sources': (),
948
- 'tags': (),
949
- 'view': fork00}),
950
- (3,
951
- {'data': (),
952
- 'form': 'meta:seen',
1090
+ 'sodes': {
1091
+ fork01layr: {
1092
+ 'form': 'meta:seen',
1093
+ 'valu': ((source22, ('it:sec:cpe', invcpe02)), 13),
1094
+ },
1095
+ },
1096
+ 'n1edges': {},
1097
+ 'n2edges': {},
1098
+ 'nodedata': {},
1099
+ 'refs': {
1100
+ fork01layr: (
1101
+ ('11f7e64a8dd8aa5f2a9b52c0e95783da4b7486452aff74dfcf80814f72507f88',
1102
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1103
+ ),
1104
+ },
1105
+ },
1106
+ {'formname': 'meta:seen',
953
1107
  'iden': metaseen03,
954
- 'layer': forklayr,
955
- 'props': {'node': ('it:sec:cpe', invcpe02),
956
- 'source': source23},
957
- 'repr': (source23, ('it:sec:cpe', invcpe02)),
1108
+ 'layers': (fork01layr,),
1109
+ 'formvalu': (source23, ('it:sec:cpe', invcpe02)),
1110
+ 'sources': (),
1111
+ 'sodes': {
1112
+ fork01layr: {
1113
+ 'form': 'meta:seen',
1114
+ 'valu': ((source23, ('it:sec:cpe', invcpe02)), 13),
1115
+ },
1116
+ },
1117
+ 'n1edges': {},
1118
+ 'n2edges': {},
1119
+ 'nodedata': {},
1120
+ 'refs': {
1121
+ fork01layr: (
1122
+ ('b209cfe6fb7167cc7dbae9df50894c2614cb9e179e5b3a4fd85fbcf7fa31a9dd',
1123
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1124
+ ),
1125
+ },
1126
+ },
1127
+ {'formname': 'meta:seen',
1128
+ 'iden': metaseen06,
1129
+ 'layers': (fork01layr,),
1130
+ 'formvalu': (source23, ('it:sec:cpe', invcpe04)),
958
1131
  'sources': (),
959
- 'tags': (),
960
- 'view': fork00}),
961
- (4,
962
- {'data': (('cpe22', 'invalid'), ('cpe23', 'invalid')),
963
- 'form': 'it:sec:cpe',
1132
+ 'sodes': {
1133
+ fork01layr: {
1134
+ 'form': 'meta:seen',
1135
+ 'valu': ((source23, ('it:sec:cpe', invcpe04)), 13),
1136
+ },
1137
+ },
1138
+ 'n1edges': {},
1139
+ 'n2edges': {},
1140
+ 'nodedata': {},
1141
+ 'refs': {
1142
+ fork01layr: (
1143
+ ('e3c389c194609a57cde68c21cac8ae1cd18e6a642e332461a3acd19138904239',
1144
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1145
+ ),
1146
+ },
1147
+ },
1148
+ {'formname': 'meta:seen',
1149
+ 'iden': metaseen01,
1150
+ 'layers': (fork01layr,),
1151
+ 'formvalu': (source23, ('it:sec:cpe', invcpe01)),
1152
+ 'sources': (),
1153
+ 'sodes': {
1154
+ fork01layr: {
1155
+ 'form': 'meta:seen',
1156
+ 'valu': ((source23, ('it:sec:cpe', invcpe01)), 13),
1157
+ },
1158
+ },
1159
+ 'n1edges': {},
1160
+ 'n2edges': {},
1161
+ 'nodedata': {},
1162
+ 'refs': {
1163
+ fork01layr: (
1164
+ ('1e0ce923f3dbd57b11d5d95cc5d6d1ccd4de4aba9b6534d57eaa0a2433af9430',
1165
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1166
+ ),
1167
+ },
1168
+ },
1169
+ {'formname': 'it:sec:cpe',
964
1170
  'iden': badcpe00,
965
- 'layer': viewlayr,
966
- 'props': {'.seen': (1577836800000, 1609459200000),
967
- 'edition': '*',
968
- 'language': '*',
969
- 'other': '*',
970
- 'part': 'a',
971
- 'product': 'openssh',
972
- 'sw_edition': '*',
973
- 'target_hw': '*',
974
- 'target_sw': '*',
975
- 'update': '*',
976
- 'v2_2': 'cpe:/a:openbsd:openssh:7.4\r\n',
977
- 'vendor': 'openbsd',
978
- 'version': '7.4',
979
- '_cpe22valid': 0,
980
- '_cpe23valid': 0},
981
- 'repr': invcpe02,
982
- 'sources': (source23,
983
- source22),
984
- 'tags': ('test', 'test.cpe', 'test.cpe.23invalid', 'test.cpe.22invalid'),
985
- 'view': view00}),
986
- (5,
987
- {'data': (),
988
- 'form': 'meta:seen',
1171
+ 'layers': tuple(sorted((fork00layr, fork03layr))),
1172
+ 'formvalu': invcpe03,
1173
+ 'sources': tuple(sorted((source22, source23))),
1174
+ 'n1edges': {
1175
+ fork00layr: (
1176
+ ('refs', 'f0315900f365f45f2e027edc66ed8477d8661dad501d51f3ac8067c36565f07c'),
1177
+ ),
1178
+ },
1179
+ 'n2edges': {
1180
+ fork01layr: (
1181
+ ('seen', '051d93252abe655e43265b89149b6a2d5a8f5f2df33b56c986ab8671c081e394', 'meta:source'),
1182
+ ('seen', '6db5f4049ac1916928f41cc5928fa60cd8fe80c453c6b2325324874a184e77da', 'meta:source'),
1183
+ ),
1184
+ },
1185
+ 'nodedata': {
1186
+ fork00layr: (
1187
+ ('cpe22', 'invalid'),
1188
+ ('cpe23', 'invalid'),
1189
+ ),
1190
+ },
1191
+ 'sodes': {
1192
+ fork00layr: {
1193
+ 'form': 'it:sec:cpe',
1194
+ 'props': {
1195
+ '.seen': ((1577836800000, 1609459200000), 12),
1196
+ '_cpe22valid': (0, 2),
1197
+ '_cpe23valid': (0, 2),
1198
+ },
1199
+ 'tagprops': {
1200
+ 'test.tagprop': {
1201
+ 'score': (0, 9),
1202
+ },
1203
+ },
1204
+ 'tags': {
1205
+ 'test': (None, None),
1206
+ 'test.cpe': (None, None),
1207
+ 'test.cpe.22invalid': (None, None),
1208
+ 'test.cpe.23invalid': (None, None),
1209
+ 'test.cpe.ival': (1577836800000, 1609459200000),
1210
+ 'test.tagprop': (None, None),
1211
+ },
1212
+ 'valu': ('cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*', 1),
1213
+ },
1214
+ fork03layr: {
1215
+ 'form': 'it:sec:cpe',
1216
+ 'valu': ('cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*', 1),
1217
+ },
1218
+ },
1219
+ 'refs': {
1220
+ fork01layr: (
1221
+ ('9742664e24fe1a3a37d871b1f62af27453c2945b98f421d753db8436e9a44cc9',
1222
+ ('it:prod:soft', 'cpe', 'it:sec:cpe', False, False)),
1223
+ ('16e3289346a258c3e3073affad490c1d6ebf1d01295aacc489cdb24658ebc6e7',
1224
+ ('_ext:model:form', 'cpe', 'it:sec:cpe', False, False)),
1225
+ ('7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7',
1226
+ ('inet:flow', 'dst:cpes', 'it:sec:cpe', True, False)),
1227
+ ('7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7',
1228
+ ('inet:flow', 'src:cpes', 'it:sec:cpe', True, False)),
1229
+ ('81973208bc0f5b99250e4cda7889c66e0573c0573bc2a279083d23426ba3c74d',
1230
+ ('meta:seen', 'node', 'ndef', False, True)),
1231
+ ('85bfc442d87a64a8e75d4ff2831281fb156317767612eef9b75c271ff162c4d9',
1232
+ ('meta:seen', 'node', 'ndef', False, True)),
1233
+ ),
1234
+ fork00layr: (
1235
+ ('5fddf1b5fa06aa8a39a1eb297712cecf9ca146764c4d6e5c79296b9e9978d2c3',
1236
+ ('risk:vulnerable', 'node', 'ndef', False, False)),
1237
+ ),
1238
+ },
1239
+ },
1240
+ {'formname': 'meta:seen',
989
1241
  'iden': metaseen04,
990
- 'layer': forklayr,
991
- 'props': {'node': ('it:sec:cpe', invcpe03),
992
- 'source': source23},
993
- 'repr': (source23, ('it:sec:cpe', invcpe03)),
1242
+ 'layers': (fork01layr,),
1243
+ 'formvalu': (source23, ('it:sec:cpe', invcpe03)),
994
1244
  'sources': (),
995
- 'tags': (),
996
- 'view': fork00}),
997
- (6,
998
- {'data': (),
999
- 'form': 'meta:seen',
1245
+ 'sodes': {
1246
+ fork01layr: {
1247
+ 'form': 'meta:seen',
1248
+ 'valu': ((source23, ('it:sec:cpe', invcpe03)), 13),
1249
+ },
1250
+ },
1251
+ 'n1edges': {},
1252
+ 'n2edges': {},
1253
+ 'nodedata': {},
1254
+ 'refs': {
1255
+ fork01layr: (
1256
+ ('6d09c45666b3a14bf9d298079344d01c079e474423307da553d65ad9917556ae',
1257
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1258
+ ),
1259
+ },
1260
+ },
1261
+ {'formname': 'meta:seen',
1000
1262
  'iden': metaseen05,
1001
- 'layer': forklayr,
1002
- 'props': {'node': ('it:sec:cpe', invcpe03),
1003
- 'source': source22},
1004
- 'repr': (source22, ('it:sec:cpe', invcpe03)),
1263
+ 'layers': (fork01layr,),
1264
+ 'formvalu': (source22, ('it:sec:cpe', invcpe03)),
1005
1265
  'sources': (),
1006
- 'tags': (),
1007
- 'view': fork00}),
1008
- (7,
1009
- {'data': (('cpe22', 'invalid'), ('cpe23', 'invalid')),
1010
- 'form': 'it:sec:cpe',
1011
- 'iden': badcpe01,
1012
- 'layer': viewlayr,
1013
- 'props': {'.seen': (1577836800000, 1609459200000),
1014
- 'edition': '*',
1015
- 'language': '*',
1016
- 'other': '*',
1017
- 'part': 'a',
1018
- 'product': 'openssh',
1019
- 'sw_edition': '*',
1020
- 'target_hw': '*',
1021
- 'target_sw': '*',
1022
- 'update': '*',
1023
- 'v2_2': 'cpe:/a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2',
1024
- 'vendor': 'openbsd',
1025
- 'version': '8.2p1 ubuntu-4ubuntu0.2',
1026
- '_cpe22valid': 0,
1027
- '_cpe23valid': 0},
1028
- 'repr': invcpe03,
1029
- 'sources': (source23,
1030
- source22),
1031
- 'tags': ('test', 'test.cpe', 'test.cpe.23invalid', 'test.cpe.22invalid'),
1032
- 'view': view00}),
1033
- (8,
1034
- {'data': (),
1035
- 'form': 'meta:seen',
1036
- 'iden': metaseen06,
1037
- 'layer': forklayr,
1038
- 'props': {'node': ('it:sec:cpe', invcpe04),
1039
- 'source': source23},
1040
- 'repr': (source23, ('it:sec:cpe', invcpe04)),
1266
+ 'sodes': {
1267
+ fork01layr: {
1268
+ 'form': 'meta:seen',
1269
+ 'valu': ((source22, ('it:sec:cpe', invcpe03)), 13),
1270
+ },
1271
+ },
1272
+ 'n1edges': {},
1273
+ 'n2edges': {},
1274
+ 'nodedata': {},
1275
+ 'refs': {
1276
+ fork01layr: (
1277
+ ('208ea1b5593aff3c9cb51c19374616fcd103ea2f554f0dd2a13652aadabb82ae',
1278
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1279
+ ),
1280
+ },
1281
+ },
1282
+ {'formname': 'meta:seen',
1283
+ 'iden': metaseen00,
1284
+ 'layers': (fork01layr,),
1285
+ 'formvalu': (source23, ('it:sec:cpe', invcpe00)),
1041
1286
  'sources': (),
1042
- 'tags': (),
1043
- 'view': fork00}),
1044
- (9,
1045
- {'data': (),
1046
- 'form': 'meta:seen',
1287
+ 'sodes': {
1288
+ fork01layr: {
1289
+ 'form': 'meta:seen',
1290
+ 'valu': ((source23, ('it:sec:cpe', invcpe00)), 13),
1291
+ },
1292
+ },
1293
+ 'n1edges': {},
1294
+ 'n2edges': {},
1295
+ 'nodedata': {},
1296
+ 'refs': {
1297
+ fork01layr: (
1298
+ ('86288a55af26e1314ae60e12c54c02f4af2e22ed1580166b39f5352762856335',
1299
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1300
+ ),
1301
+ },
1302
+ },
1303
+ {'formname': 'meta:seen',
1047
1304
  'iden': metaseen07,
1048
- 'layer': forklayr,
1049
- 'props': {'node': ('it:sec:cpe', invcpe05),
1050
- 'source': source23},
1051
- 'repr': (source23, ('it:sec:cpe', invcpe05)),
1305
+ 'layers': (fork01layr,),
1306
+ 'formvalu': (source23, ('it:sec:cpe', invcpe05)),
1052
1307
  'sources': (),
1053
- 'tags': (),
1054
- 'view': fork00}),
1055
- ])
1308
+ 'sodes': {
1309
+ fork01layr: {
1310
+ 'form': 'meta:seen',
1311
+ 'valu': ((source23, ('it:sec:cpe', invcpe05)), 13),
1312
+ },
1313
+ },
1314
+ 'n1edges': {},
1315
+ 'n2edges': {},
1316
+ 'nodedata': {},
1317
+ 'refs': {
1318
+ fork01layr: (
1319
+ ('53ad1502b6f6de3d9d4efe72cc101cd3889e47323ac8db5e3fd39ae68c72f141',
1320
+ ('it:sec:vuln:scan:result', 'asset', 'ndef', False, False)),
1321
+ ),
1322
+ },
1323
+ },
1324
+ ]
1325
+
1326
+ for item in expected:
1327
+ self.isin(item, nodesq)
1328
+
1329
+ # There should be nothing in the default view
1330
+ nodes = await core.nodes('.created')
1331
+ self.len(0, nodes)
1056
1332
 
1057
- q = '''
1058
- $ret = ([])
1059
- $q = $lib.queue.get('model_0_2_28:nodes:refs')
1060
- for $ii in $lib.range(($q.size())) {
1061
- $ret.append($q.get($ii, cull=(false), wait=(false)))
1062
- }
1063
- fini { return($ret) }
1064
- '''
1065
- refsq = await core.callStorm(q)
1066
- self.eq(refsq, [
1067
- (0,
1068
- {'iden': metaseen00,
1069
- 'refs': ({'iden': '86288a55af26e1314ae60e12c54c02f4af2e22ed1580166b39f5352762856335',
1070
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1071
- 'view': fork00}),
1072
- (1,
1073
- {'iden': metaseen01,
1074
- 'refs': ({'iden': '1e0ce923f3dbd57b11d5d95cc5d6d1ccd4de4aba9b6534d57eaa0a2433af9430',
1075
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1076
- 'view': fork00}),
1077
- (2,
1078
- {'iden': badcpe00,
1079
- 'refs': ({'iden': 'd2c0737b821ba0a699e1ff168e2bf1677590dbf677ce7e5c02894f8868ae080a',
1080
- 'refinfo': ('risk:vulnerable', 'node', 'ndef', False)},),
1081
- 'view': view00}),
1082
- (3,
1083
- {'iden': badcpe00,
1084
- 'refs': ({'iden': '7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7',
1085
- 'refinfo': ('inet:flow', 'src:cpes', 'it:sec:cpe', True)},),
1086
- 'view': fork00}),
1087
- (4,
1088
- {'iden': metaseen02,
1089
- 'refs': ({'iden': '11f7e64a8dd8aa5f2a9b52c0e95783da4b7486452aff74dfcf80814f72507f88',
1090
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1091
- 'view': fork00}),
1092
- (5,
1093
- {'iden': metaseen03,
1094
- 'refs': ({'iden': 'b209cfe6fb7167cc7dbae9df50894c2614cb9e179e5b3a4fd85fbcf7fa31a9dd',
1095
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1096
- 'view': fork00}),
1097
- (6,
1098
- {'iden': badcpe01,
1099
- 'refs': ({'iden': '5fddf1b5fa06aa8a39a1eb297712cecf9ca146764c4d6e5c79296b9e9978d2c3',
1100
- 'refinfo': ('risk:vulnerable', 'node', 'ndef', False)},),
1101
- 'view': view00}),
1102
- (7,
1103
- {'iden': badcpe01,
1104
- 'refs': ({'iden': '9742664e24fe1a3a37d871b1f62af27453c2945b98f421d753db8436e9a44cc9',
1105
- 'refinfo': ('it:prod:soft', 'cpe', 'it:sec:cpe', False)},),
1106
- 'view': fork00}),
1107
- (8,
1108
- {'iden': badcpe01,
1109
- 'refs': ({'iden': '16e3289346a258c3e3073affad490c1d6ebf1d01295aacc489cdb24658ebc6e7',
1110
- 'refinfo': ('_ext:model:form', 'cpe', 'it:sec:cpe', False)},),
1111
- 'view': fork00}),
1112
- (9,
1113
- {'iden': badcpe01,
1114
- 'refs': ({'iden': '7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7',
1115
- 'refinfo': ('inet:flow', 'src:cpes', 'it:sec:cpe', True)},),
1116
- 'view': fork00}),
1117
- (10,
1118
- {'iden': metaseen04,
1119
- 'refs': ({'iden': '6d09c45666b3a14bf9d298079344d01c079e474423307da553d65ad9917556ae',
1120
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1121
- 'view': fork00}),
1122
- (11,
1123
- {'iden': metaseen05,
1124
- 'refs': ({'iden': '208ea1b5593aff3c9cb51c19374616fcd103ea2f554f0dd2a13652aadabb82ae',
1125
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1126
- 'view': fork00}),
1127
- (12,
1128
- {'iden': metaseen06,
1129
- 'refs': ({'iden': 'e3c389c194609a57cde68c21cac8ae1cd18e6a642e332461a3acd19138904239',
1130
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1131
- 'view': fork00}),
1132
- (13,
1133
- {'iden': metaseen07,
1134
- 'refs': ({'iden': '53ad1502b6f6de3d9d4efe72cc101cd3889e47323ac8db5e3fd39ae68c72f141',
1135
- 'refinfo': ('it:sec:vuln:scan:result', 'asset', 'ndef', False)},),
1136
- 'view': fork00}),
1137
- ])
1138
-
1139
- riskvuln00 = await core.callStorm('risk:vuln return($node.iden())')
1140
- q = '''
1141
- $ret = ([])
1142
- $q = $lib.queue.get('model_0_2_28:nodes:edges')
1143
- for $ii in $lib.range(($q.size())) {
1144
- $ret.append($q.get($ii, cull=(false), wait=(false)))
1145
- }
1146
- fini { return($ret) }
1147
- '''
1148
- edgesq = await core.callStorm(q)
1149
- self.eq(edgesq, [
1150
- (0,
1151
- {'direction': 'n1',
1152
- 'edges': (('refs', riskvuln00),),
1153
- 'iden': badcpe00,
1154
- 'view': view00}),
1155
- (1,
1156
- {'direction': 'n2',
1157
- 'edges': (('seen', source23iden), ('seen', source22iden)),
1158
- 'iden': badcpe00,
1159
- 'view': fork00}),
1160
- (2,
1161
- {'direction': 'n1',
1162
- 'edges': (('refs', riskvuln00),),
1163
- 'iden': badcpe01,
1164
- 'view': view00}),
1165
- (3,
1166
- {'direction': 'n2',
1167
- 'edges': (('seen', source23iden), ('seen', source22iden)),
1168
- 'iden': badcpe01,
1169
- 'view': fork00}),
1170
- ])
1333
+ async with self.getRegrCore('model-cpe-migration') as core:
1171
1334
 
1172
- async def test_modelrev_0_2_29(self):
1173
- async with self.getRegrCore('model-0.2.29') as core:
1174
- self.len(2, await core.nodes('ou:industry:type:taxonomy'))
1335
+ riskvuln = s_common.ehex(s_common.buid(('risk:vuln', s_common.guid(('risk', 'vuln')))))
1175
1336
 
1176
- async def test_modelrev_0_2_30(self):
1177
- async with self.getRegrCore('model-0.2.30') as core:
1178
- q = '''
1179
- inet:ipv4=192.0.0.0 inet:ipv4=192.0.0.8 inet:ipv4=192.0.0.9 inet:ipv4=192.0.0.10 inet:ipv4=192.0.0.255
1180
- '''
1181
- nodes = await core.nodes(q)
1182
- typz = [node.get('type') for node in nodes]
1183
- self.eq(typz, ['private', 'private', 'unicast', 'unicast', 'private'])
1337
+ views = {view.info.get('name'): view for view in core.listViews()}
1338
+ self.len(5, views)
1184
1339
 
1185
- q = '''
1186
- inet:ipv6="64:ff9b:1::" inet:ipv6="64:ff9b:1::1" inet:ipv6="64:ff9b:1::ffff" inet:ipv6="64:ff9b:1::ffff:1"
1187
- '''
1188
- nodes = await core.nodes(q)
1189
- typz = [node.get('type') for node in nodes]
1190
- self.eq(typz, ['private', 'private', 'private', 'private'])
1340
+ fork02 = views.get('fork02').iden
1341
+ infork02 = {'view': fork02}
1342
+
1343
+ # Normal lift will go through the views
1344
+ nodes = await core.nodes('it:sec:cpe:vendor=01generator', opts=infork02)
1345
+ self.len(1, nodes)
1346
+ self.eq(nodes[0].get('v2_2'), 'cpe:/a:01generator:pireospay:-::~~~prestashop~~')
1191
1347
 
1348
+ # The v2_2 floating props in this view were removed because the underlying nodes were completely invalid and
1349
+ # could not be migrated
1192
1350
  q = '''
1193
- inet:ipv6="2002::" inet:ipv6="2002::1" inet:ipv6="2002::fffe" inet:ipv6="2002::ffff"
1351
+ $nodes = ([])
1352
+
1353
+ for $n in $lib.view.get().layers.0.liftByProp("it:sec:cpe:v2_2") {
1354
+ $nodes.append($n)
1355
+ }
1356
+
1357
+ return($nodes)
1194
1358
  '''
1195
- nodes = await core.nodes(q)
1196
- typz = [node.get('type') for node in nodes]
1197
- self.eq(typz, ['private', 'private', 'private', 'private'])
1359
+ nodes = await core.callStorm(q, opts=infork02)
1360
+ self.len(0, nodes)
1198
1361
 
1199
- q = 'inet:ipv6="2001:1::1/128" inet:ipv6="2001:1::2/128"'
1200
- nodes = await core.nodes(q)
1201
- typz = [node.get('type') for node in nodes]
1202
- self.eq(typz, ['unicast', 'unicast'])
1362
+ nodes = await core.nodes('meta:source:name=cpe.22.valid', opts=infork02)
1363
+ self.len(1, nodes)
1364
+ meta22valid = nodes[0]
1203
1365
 
1204
- q = 'inet:ipv6="2001:3::" inet:ipv6="2001:3::1" inet:ipv6="2001:3::ffff"'
1205
- nodes = await core.nodes(q)
1206
- typz = [node.get('type') for node in nodes]
1207
- self.eq(typz, ['unicast', 'unicast', 'unicast'])
1366
+ nodes = await core.nodes('meta:source:name=cpe.22.invalid', opts=infork02)
1367
+ self.len(1, nodes)
1368
+ meta22invalid = nodes[0]
1208
1369
 
1209
- q = 'inet:ipv6="2001:4:112::" inet:ipv6="2001:4:112::1" inet:ipv6="2001:4:112::ffff"'
1210
- nodes = await core.nodes(q)
1211
- typz = [node.get('type') for node in nodes]
1212
- self.eq(typz, ['unicast', 'unicast', 'unicast'])
1370
+ nodes = await core.nodes('meta:source:name=cpe.22.wasinvalid', opts=infork02)
1371
+ self.len(1, nodes)
1372
+ meta22wasinvalid = nodes[0]
1213
1373
 
1214
- q = 'inet:ipv6="2001:20::" inet:ipv6="2001:20::1" inet:ipv6="2001:20::ffff"'
1215
- nodes = await core.nodes(q)
1216
- typz = [node.get('type') for node in nodes]
1217
- self.eq(typz, ['unicast', 'unicast', 'unicast'])
1374
+ nodes = await core.nodes('meta:source:name=cpe.23.valid', opts=infork02)
1375
+ self.len(1, nodes)
1376
+ meta23valid = nodes[0]
1218
1377
 
1219
- q = 'inet:ipv6="2001:30::" inet:ipv6="2001:30::1" inet:ipv6="2001:30::ffff"'
1220
- nodes = await core.nodes(q)
1221
- typz = [node.get('type') for node in nodes]
1222
- self.eq(typz, ['unicast', 'unicast', 'unicast'])
1378
+ nodes = await core.nodes('meta:source:name=cpe.23.invalid', opts=infork02)
1379
+ self.len(1, nodes)
1380
+ meta23invalid = nodes[0]
1381
+
1382
+ nodes = await core.nodes('meta:source:name=cpe.23.wasinvalid', opts=infork02)
1383
+ self.len(1, nodes)
1384
+ meta23wasinvalid = nodes[0]
1385
+
1386
+ nodes = await core.nodes('risk:vuln', opts=infork02)
1387
+ self.len(1, nodes)
1388
+ riskvuln = nodes[0]
1389
+
1390
+ nodes = await core.nodes('it:sec:cpe#test.cpe.23valid +#test.cpe.22invalid', opts=infork02)
1391
+ self.len(3, nodes)
1392
+ for node in nodes:
1393
+
1394
+ self.true(node.get('_cpe22valid'))
1395
+ self.true(node.get('_cpe23valid'))
1396
+ self.eq(node.get('.seen'), (1577836800000, 1672531200001)) # .seen = (2020, 2023)
1397
+ self.isin('test.cpe.22valid', node.tags)
1398
+ self.isin('test.tagprop', node.tags)
1399
+ self.eq(['score'], node.getTagProps('test.tagprop'))
1400
+ self.eq(11, node.getTagProp('test.tagprop', 'score'))
1401
+
1402
+ nodedata = await s_tests.alist(node.iterData())
1403
+ self.eq(nodedata, [('cpe22', 'wasinvalid'), ('cpe23', 'valid')])
1404
+
1405
+ n1s = await s_tests.alist(node.iterEdgesN1())
1406
+ self.sorteq(n1s, [
1407
+ ('refs', meta23valid.iden()),
1408
+ ('refs', meta22wasinvalid.iden()),
1409
+ ('refs', riskvuln.iden())
1410
+ ])
1411
+
1412
+ for n in (meta23valid, meta22wasinvalid, riskvuln):
1413
+ n2s = await s_tests.alist(n.iterEdgesN2())
1414
+ self.isin(('refs', node.iden()), n2s)
1415
+
1416
+ n2s = await s_tests.alist(node.iterEdgesN2())
1417
+ self.sorteq(n2s, [
1418
+ ('seen', meta22invalid.iden()),
1419
+ ('seen', meta22wasinvalid.iden()),
1420
+ ('seen', meta23valid.iden())
1421
+ ])
1422
+
1423
+ for n in (meta22invalid, meta22wasinvalid, meta23valid):
1424
+ n1s = await s_tests.alist(n.iterEdgesN1())
1425
+ self.isin(('seen', node.iden()), n1s)
1426
+
1427
+ nodes = await core.nodes('it:sec:cpe#test.cpe.22valid +#test.cpe.23invalid', opts=infork02)
1428
+ self.len(4, nodes)
1429
+ for node in nodes:
1430
+ self.true(node.get('_cpe22valid'))
1431
+ self.true(node.get('_cpe23valid'))
1432
+ self.eq(node.get('.seen'), (1577836800000, 1704067200001)) # .seen = (2020, 2024)
1433
+ self.isin('test.cpe.23valid', node.tags)
1434
+ self.isin('test.tagprop', node.tags)
1435
+ self.eq(['score'], node.getTagProps('test.tagprop'))
1436
+ self.eq(11, node.getTagProp('test.tagprop', 'score'))
1437
+
1438
+ nodedata = await s_tests.alist(node.iterData())
1439
+ self.eq(nodedata, [('cpe23', 'wasinvalid'), ('cpe22', 'valid')])
1440
+
1441
+ n1s = await s_tests.alist(node.iterEdgesN1())
1442
+ self.sorteq(n1s, [
1443
+ ('refs', meta22valid.iden()),
1444
+ ('refs', meta23wasinvalid.iden()),
1445
+ ('refs', riskvuln.iden())
1446
+ ])
1447
+
1448
+ for n in (meta22valid, meta23wasinvalid, riskvuln):
1449
+ n2s = await s_tests.alist(n.iterEdgesN2())
1450
+ self.isin(('refs', node.iden()), n2s)
1451
+
1452
+ n2s = await s_tests.alist(node.iterEdgesN2())
1453
+ self.sorteq(n2s, [
1454
+ ('seen', meta23invalid.iden()),
1455
+ ('seen', meta23wasinvalid.iden()),
1456
+ ('seen', meta22valid.iden())
1457
+ ])
1458
+
1459
+ for n in (meta23invalid, meta23wasinvalid, meta22valid):
1460
+ n1s = await s_tests.alist(n.iterEdgesN1())
1461
+ self.isin(('seen', node.iden()), n1s)
1462
+
1463
+ # There should be nothing in the default view
1464
+ nodes = await core.nodes('.created')
1465
+ self.len(0, nodes)
1466
+
1467
+ orig = s_spooled.Spooled.__anit__
1468
+ for maxval in (s_spooled.MAX_SPOOL_SIZE, 1):
1469
+
1470
+ async def __anit__(self, dirn=None, size=s_spooled.MAX_SPOOL_SIZE, cell=None):
1471
+ await orig(self, dirn=dirn, size=maxval, cell=cell)
1472
+
1473
+ with mock.patch('synapse.lib.spooled.Spooled.__anit__', __anit__):
1474
+ async with self.getRegrCore('model-cpe-migration') as core:
1475
+ # Make sure the mock worked
1476
+ migration = await s_modelrev.ModelMigration_0_2_31.anit(core, [])
1477
+ self.eq(migration.nodes.size, maxval)
1478
+ self.eq(migration.todos.size, maxval)
1479
+
1480
+ riskvuln = s_common.ehex(s_common.buid(('risk:vuln', s_common.guid(('risk', 'vuln')))))
1481
+
1482
+ views = {view.info.get('name'): view for view in core.listViews()}
1483
+ self.len(5, views)
1484
+
1485
+ fork00 = views.get('fork00').iden
1486
+ infork00 = {'view': fork00}
1487
+
1488
+ fork01 = views.get('fork01').iden
1489
+ infork01 = {'view': fork01}
1490
+
1491
+ fork02 = views.get('fork02').iden
1492
+ infork02 = {'view': fork02}
1493
+
1494
+ q = '''
1495
+ $ret = ([])
1496
+ for ($offs, $form, $valu, $sources) in $lib.model.migration.s.model_0_2_31.listNodes() {
1497
+ $srcs = ([])
1498
+ for $src in $lib.sorted($sources) { $srcs.append($src) }
1499
+ $ret.append(($form, $valu, $srcs))
1500
+ }
1501
+ return($ret)
1502
+ '''
1503
+ nodelist = await core.callStorm(q)
1504
+ expected = [
1505
+ ('meta:seen', (
1506
+ '008af0047a8350287cde7abe31a7c706',
1507
+ ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*')
1508
+ ),
1509
+ (),
1510
+ ),
1511
+ ('meta:seen', (
1512
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1513
+ ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:7.4\r\n:*:*:*:*:*:*:*')
1514
+ ),
1515
+ (),
1516
+ ),
1517
+ ('meta:seen', (
1518
+ '008af0047a8350287cde7abe31a7c706',
1519
+ ('it:sec:cpe', 'cpe:2.3:a:%40ianwalter:merge:*:*:*:*:*:*:*:*')
1520
+ ),
1521
+ (),
1522
+ ),
1523
+ ('meta:seen', (
1524
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1525
+ ('it:sec:cpe', 'cpe:2.3:a:%40ianwalter:merge:*:*:*:*:*:*:*:*')
1526
+ ),
1527
+ (),
1528
+ ),
1529
+ ('meta:seen', (
1530
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1531
+ ('it:sec:cpe', 'cpe:2.3:h:d\\-link:dir\\-850l:*:*:*:*:*:*:*:*')
1532
+ ),
1533
+ (),
1534
+ ),
1535
+ ('meta:seen', (
1536
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1537
+ ('it:sec:cpe', 'cpe:2.3:a:acurax:under_construction_%2f_maintenance_mode:-::~~~wordpress~~:*:*:*:*:*')
1538
+ ),
1539
+ (),
1540
+ ),
1541
+ ('it:sec:cpe',
1542
+ 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*',
1543
+ tuple(sorted((source22, source23))),
1544
+ ),
1545
+ ('meta:seen', (
1546
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1547
+ ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
1548
+ ),
1549
+ (),
1550
+ ),
1551
+ ('meta:seen', (
1552
+ '008af0047a8350287cde7abe31a7c706',
1553
+ ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
1554
+ ),
1555
+ (),
1556
+ ),
1557
+ ('meta:seen', (
1558
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1559
+ ('it:sec:cpe', 'cpe:2.3:a:10web:social_feed_for_instagram:1.0.0::~~premium~wordpress~~:*:*:*:*:*')
1560
+ ),
1561
+ (),
1562
+ ),
1563
+ ('meta:seen', (
1564
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1565
+ ('it:sec:cpe', 'cpe:2.3:o:zyxel:nas326_firmware:5.21%28aazf.14%29c0:*:*:*:*:*:*:*')
1566
+ ),
1567
+ (),
1568
+ ),
1569
+ ]
1570
+ for item in expected:
1571
+ self.isin(item, nodelist)
1572
+
1573
+ cpeidx = nodelist.index(
1574
+ ('it:sec:cpe',
1575
+ 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*',
1576
+ tuple(sorted((source22, source23)))
1577
+ )
1578
+ )
1579
+ metaidx = nodelist.index(
1580
+ ('meta:seen', (
1581
+ 'a7a4739e0a52674df0fa3a8226de0c3f',
1582
+ ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*')
1583
+ ),
1584
+ (),
1585
+ )
1586
+ )
1587
+
1588
+ q = '''
1589
+ $ret = ([])
1590
+ for ($offs, $form, $valu, $sources) in $lib.model.migration.s.model_0_2_31.listNodes(form=it:sec:cpe, source=$source22) {
1591
+ $srcs = ([])
1592
+ for $src in $lib.sorted($sources) { $srcs.append($src) }
1593
+ $ret.append(($form, $valu, $srcs))
1594
+ }
1595
+ return($ret)
1596
+ '''
1597
+ opts = {'vars': {'source22': source22}}
1598
+ nodelist = await core.callStorm(q, opts=opts)
1599
+ self.len(1, nodelist)
1600
+ self.eq(nodelist[0],
1601
+ ('it:sec:cpe',
1602
+ 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*',
1603
+ tuple(sorted((source22, source23))),
1604
+ ),
1605
+ )
1606
+
1607
+ msgs = await core.stormlist('$lib.model.migration.s.model_0_2_31.printNode((200))')
1608
+ self.stormIsInWarn('Queued node with offset 200 not found.', msgs)
1609
+
1610
+ msgs = await core.stormlist('$lib.model.migration.s.model_0_2_31.repairNode((200), "")')
1611
+ self.stormIsInWarn('Queued node with offset 200 not found.', msgs)
1612
+
1613
+ msgs = await core.stormlist(f'$lib.model.migration.s.model_0_2_31.printNode(({cpeidx}))')
1614
+ self.stormHasNoWarnErr(msgs)
1615
+
1616
+ output = textwrap.dedent(f'''
1617
+ it:sec:cpe='cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*'
1618
+ layer: {fork03layr}
1619
+ layer: {fork00layr}
1620
+ :_cpe22valid = 0
1621
+ :_cpe23valid = 0
1622
+ .seen = (2020/01/01 00:00:00.000, 2021/01/01 00:00:00.000)
1623
+ #test
1624
+ #test.cpe
1625
+ #test.cpe.ival = (2020/01/01 00:00:00.000, 2021/01/01 00:00:00.000)
1626
+ #test.cpe.23invalid
1627
+ #test.cpe.22invalid
1628
+ #test.tagprop
1629
+ #test.tagprop:score = 0
1630
+ sources: ['008af0047a8350287cde7abe31a7c706', 'a7a4739e0a52674df0fa3a8226de0c3f']
1631
+ refs:
1632
+ layer: {fork01layr}
1633
+ - it:prod:soft:cpe (iden: 9742664e24fe1a3a37d871b1f62af27453c2945b98f421d753db8436e9a44cc9)
1634
+ - _ext:model:form:cpe (iden: 16e3289346a258c3e3073affad490c1d6ebf1d01295aacc489cdb24658ebc6e7)
1635
+ - inet:flow:dst:cpes (iden: 7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7)
1636
+ - inet:flow:src:cpes (iden: 7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7)
1637
+ - meta:seen:node (iden: 81973208bc0f5b99250e4cda7889c66e0573c0573bc2a279083d23426ba3c74d)
1638
+ - meta:seen:node (iden: 85bfc442d87a64a8e75d4ff2831281fb156317767612eef9b75c271ff162c4d9)
1639
+ layer: {fork00layr}
1640
+ - risk:vulnerable:node (iden: 5fddf1b5fa06aa8a39a1eb297712cecf9ca146764c4d6e5c79296b9e9978d2c3)
1641
+ edges:
1642
+ -(refs)> f0315900f365f45f2e027edc66ed8477d8661dad501d51f3ac8067c36565f07c
1643
+ <(seen)- 051d93252abe655e43265b89149b6a2d5a8f5f2df33b56c986ab8671c081e394
1644
+ <(seen)- 6db5f4049ac1916928f41cc5928fa60cd8fe80c453c6b2325324874a184e77da
1645
+ ''')[1:-1]
1646
+ self.stormIsInPrint(output, msgs)
1647
+
1648
+ oldcpe = 'cpe:2.3:a:openbsd:openssh:8.2p1 ubuntu-4ubuntu0.2:*:*:*:*:*:*:*'
1649
+ newcpe = 'cpe:2.3:a:openbsd:openssh:8.2p1:*:*:*:*:*:*:*'
1650
+
1651
+ nodes = await core.nodes('_ext:model:form="22i-23i"', opts=infork01)
1652
+ self.len(1, nodes)
1653
+ self.none(nodes[0].get('cpe'))
1654
+
1655
+ nodes = await core.nodes(f'risk:vulnerable=(22invalid, 23invalid, (it:sec:cpe, "{oldcpe}"))', opts=infork00)
1656
+ self.len(1, nodes)
1657
+ self.none(nodes[0].get('node'))
1658
+
1659
+ nodes = await core.nodes('inet:flow=(flow, 22i, 23i)', opts=infork01)
1660
+ self.len(1, nodes)
1661
+ self.none(nodes[0].get('dst:cpes'))
1662
+ self.notin(newcpe, nodes[0].get('src:cpes'))
1663
+
1664
+ msgs = await core.stormlist(f'$lib.model.migration.s.model_0_2_31.repairNode(({cpeidx}), "{newcpe}")')
1665
+ self.stormHasNoWarnErr(msgs)
1666
+
1667
+ # Repair node should be idempotent
1668
+ msgs = await core.stormlist(f'$lib.model.migration.s.model_0_2_31.repairNode(({cpeidx}), "{newcpe}", $lib.true)')
1669
+ self.stormHasNoWarnErr(msgs)
1670
+
1671
+ nodes = await core.nodes('it:sec:cpe:vendor=openbsd +:version="8.2p1"', opts=infork00)
1672
+ self.len(1, nodes)
1673
+ self.false(nodes[0].get('_cpe22valid'))
1674
+ self.false(nodes[0].get('_cpe23valid'))
1675
+ self.eq(nodes[0].get('.seen'), (1577836800000, 1609459200000))
1676
+ self.eq(nodes[0].get('edition'), '*')
1677
+ self.eq(nodes[0].get('language'), '*')
1678
+ self.eq(nodes[0].get('other'), '*')
1679
+ self.eq(nodes[0].get('part'), 'a')
1680
+ self.eq(nodes[0].get('product'), 'openssh')
1681
+ self.eq(nodes[0].get('sw_edition'), '*')
1682
+ self.eq(nodes[0].get('target_hw'), '*')
1683
+ self.eq(nodes[0].get('target_sw'), '*')
1684
+ self.eq(nodes[0].get('update'), '*')
1685
+ self.eq(nodes[0].get('vendor'), 'openbsd')
1686
+ self.eq(nodes[0].get('version'), '8.2p1')
1687
+ self.eq(nodes[0].get('v2_2'), 'cpe:/a:openbsd:openssh:8.2p1')
1688
+ self.isin('test.cpe.22invalid', nodes[0].tags)
1689
+ self.isin('test.cpe.23invalid', nodes[0].tags)
1690
+ self.isin('test.tagprop', nodes[0].tags)
1691
+ self.eq(nodes[0].tagprops['test.tagprop'], {'score': 0})
1692
+
1693
+ edges = await s_tests.alist(nodes[0].iterEdgesN1())
1694
+ self.len(1, edges)
1695
+ self.eq(edges, [('refs', riskvuln)])
1696
+
1697
+ edges = await s_tests.alist(nodes[0].iterEdgesN2())
1698
+ self.len(0, edges)
1699
+
1700
+ nodedata = await s_tests.alist(nodes[0].iterData())
1701
+ self.eq(nodedata, [('cpe22', 'invalid'), ('cpe23', 'invalid')])
1702
+
1703
+ nodes = await core.nodes('it:sec:cpe:vendor=openbsd +:version="8.2p1"', opts=infork01)
1704
+ self.len(1, nodes)
1705
+
1706
+ edges = await s_tests.alist(nodes[0].iterEdgesN1())
1707
+ self.len(1, edges)
1708
+ self.eq(edges, [('refs', riskvuln)])
1709
+
1710
+ edges = await s_tests.alist(nodes[0].iterEdgesN2())
1711
+ self.len(2, edges)
1712
+ self.sorteq(edges, [
1713
+ ('seen', source22iden),
1714
+ ('seen', source23iden),
1715
+ ])
1716
+
1717
+ nodes = await core.nodes('_ext:model:form="22i-23i"', opts=infork01)
1718
+ self.len(1, nodes)
1719
+ self.eq(nodes[0].get('cpe'), newcpe)
1720
+
1721
+ nodes = await core.nodes(f'risk:vulnerable=(22invalid, 23invalid, (it:sec:cpe, "{oldcpe}"))', opts=infork00)
1722
+ self.len(1, nodes)
1723
+ self.eq(nodes[0].get('node'), ('it:sec:cpe', newcpe))
1724
+
1725
+ nodes = await core.nodes('inet:flow=(flow, 22i, 23i)', opts=infork01)
1726
+ self.len(1, nodes)
1727
+ self.isin(newcpe, nodes[0].get('dst:cpes'))
1728
+ self.isin(newcpe, nodes[0].get('src:cpes'))
1729
+
1730
+ nodes = await core.nodes('it:sec:cpe:vendor=openbsd', opts=infork02)
1731
+ self.len(2, nodes)
1732
+ self.eq(nodes[0].get('v2_2'), 'cpe:/a:openbsd:openssh:8.2p1')
1733
+ self.eq(nodes[1].get('v2_2'), 'cpe:/a:openbsd:openssh_server:7.4')
1734
+
1735
+ nodes = await core.nodes('it:sec:cpe:vendor="openbsd" +:version="8.2p1" -> meta:seen', opts=infork01)
1736
+ self.len(0, nodes)
1737
+
1738
+ valu = ('a7a4739e0a52674df0fa3a8226de0c3f', ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1:*:*:*:*:*:*:*'))
1739
+ iden = '81973208bc0f5b99250e4cda7889c66e0573c0573bc2a279083d23426ba3c74d'
1740
+ q = f'$lib.model.migration.s.model_0_2_31.repairNode(({metaidx}), $valu, $lib.true)'
1741
+
1742
+ opts = {'vars': {'iden': iden, 'valu': valu}}
1743
+ msgs = await core.stormlist(q, opts=opts)
1744
+ self.stormHasNoWarnErr(msgs)
1745
+
1746
+ nodes = await core.nodes('it:sec:cpe:vendor="openbsd" +:version="8.2p1" -> meta:seen', opts=infork01)
1747
+ self.len(1, nodes)
1748
+ self.eq(nodes[0].get('source'), 'a7a4739e0a52674df0fa3a8226de0c3f')
1749
+ self.eq(nodes[0].get('node'), ('it:sec:cpe', 'cpe:2.3:a:openbsd:openssh:8.2p1:*:*:*:*:*:*:*'))
1750
+
1751
+ # Check queue status after restoring three nodes
1752
+ queues = await core.callStorm('return($lib.queue.list())')
1753
+ [q.pop('meta') for q in queues]
1754
+ self.len(1, queues)
1755
+ self.eq(queues, (
1756
+ {'name': 'model_0_2_31:nodes', 'size': 9, 'offs': 11},
1757
+ ))
1758
+
1759
+ # There should be nothing in the default view
1760
+ nodes = await core.nodes('.created')
1761
+ self.len(0, nodes)