synapse 2.168.0__py311-none-any.whl → 2.170.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (42) hide show
  1. synapse/cortex.py +99 -10
  2. synapse/datamodel.py +5 -0
  3. synapse/lib/agenda.py +3 -0
  4. synapse/lib/ast.py +70 -12
  5. synapse/lib/cell.py +83 -21
  6. synapse/lib/httpapi.py +3 -0
  7. synapse/lib/layer.py +75 -6
  8. synapse/lib/node.py +7 -0
  9. synapse/lib/snap.py +25 -5
  10. synapse/lib/storm.py +1 -1
  11. synapse/lib/stormlib/cortex.py +1 -1
  12. synapse/lib/stormlib/model.py +420 -1
  13. synapse/lib/stormtypes.py +68 -3
  14. synapse/lib/types.py +35 -0
  15. synapse/lib/version.py +2 -2
  16. synapse/lib/view.py +94 -24
  17. synapse/models/files.py +40 -0
  18. synapse/models/inet.py +8 -4
  19. synapse/models/infotech.py +355 -17
  20. synapse/tests/files/cpedata.json +525034 -0
  21. synapse/tests/test_cortex.py +99 -0
  22. synapse/tests/test_lib_agenda.py +17 -3
  23. synapse/tests/test_lib_ast.py +66 -0
  24. synapse/tests/test_lib_cell.py +133 -52
  25. synapse/tests/test_lib_layer.py +52 -1
  26. synapse/tests/test_lib_scrape.py +72 -71
  27. synapse/tests/test_lib_snap.py +16 -1
  28. synapse/tests/test_lib_storm.py +118 -0
  29. synapse/tests/test_lib_stormlib_cortex.py +27 -0
  30. synapse/tests/test_lib_stormlib_model.py +532 -0
  31. synapse/tests/test_lib_stormtypes.py +161 -14
  32. synapse/tests/test_lib_types.py +20 -0
  33. synapse/tests/test_lib_view.py +77 -0
  34. synapse/tests/test_model_files.py +51 -0
  35. synapse/tests/test_model_inet.py +63 -1
  36. synapse/tests/test_model_infotech.py +187 -26
  37. synapse/tests/utils.py +12 -0
  38. {synapse-2.168.0.dist-info → synapse-2.170.0.dist-info}/METADATA +1 -1
  39. {synapse-2.168.0.dist-info → synapse-2.170.0.dist-info}/RECORD +42 -41
  40. {synapse-2.168.0.dist-info → synapse-2.170.0.dist-info}/LICENSE +0 -0
  41. {synapse-2.168.0.dist-info → synapse-2.170.0.dist-info}/WHEEL +0 -0
  42. {synapse-2.168.0.dist-info → synapse-2.170.0.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,23 @@
1
1
  import synapse.exc as s_exc
2
2
  import synapse.common as s_common
3
3
 
4
+ import synapse.lib.node as s_node
4
5
  import synapse.lib.cache as s_cache
5
6
  import synapse.lib.stormtypes as s_stormtypes
6
7
 
8
+ import synapse.models.infotech as s_infotech
9
+
10
+ RISK_HASVULN_VULNPROPS = (
11
+ 'hardware',
12
+ 'host',
13
+ 'item',
14
+ 'org',
15
+ 'person',
16
+ 'place',
17
+ 'software',
18
+ 'spec',
19
+ )
20
+
7
21
  stormcmds = [
8
22
  {
9
23
  'name': 'model.edge.set',
@@ -310,7 +324,7 @@ class LibModel(s_stormtypes.Lib):
310
324
  {'name': 'name', 'type': 'str', 'desc': 'The name of the tag prop to retrieve.', },
311
325
  ),
312
326
  'returns': {'type': ['model:tagprop', 'null'],
313
- 'desc': 'The ``model:tagprop`` instance if the tag prop if present or null.',
327
+ 'desc': 'The ``model:tagprop`` instance of the tag prop if present or null.',
314
328
  }}},
315
329
  )
316
330
 
@@ -692,3 +706,408 @@ class LibModelDeprecated(s_stormtypes.Lib):
692
706
  todo = s_common.todo('setDeprLock', name, locked)
693
707
  gatekeys = ((self.runt.user.iden, ('model', 'deprecated', 'lock'), None),)
694
708
  await self.runt.dyncall('cortex', todo, gatekeys=gatekeys)
709
+
710
+ class MigrationEditorMixin:
711
+ '''
712
+ Mixin helpers for migrating data within an editor context.
713
+ '''
714
+
715
+ async def copyData(self, src, proto, overwrite=False):
716
+
717
+ async for name in src.iterDataKeys():
718
+ if overwrite or not await proto.hasData(name):
719
+ self.runt.layerConfirm(('node', 'data', 'set', name))
720
+ valu = await src.getData(name)
721
+ await proto.setData(name, valu)
722
+
723
+ async def copyEdges(self, editor, src, proto):
724
+
725
+ verbs = set()
726
+
727
+ async for (verb, n2iden) in src.iterEdgesN1():
728
+
729
+ if verb not in verbs:
730
+ self.runt.layerConfirm(('node', 'edge', 'add', verb))
731
+ verbs.add(verb)
732
+
733
+ if await self.runt.snap.getNodeByBuid(s_common.uhex(n2iden)) is not None:
734
+ await proto.addEdge(verb, n2iden)
735
+
736
+ dstiden = proto.iden()
737
+
738
+ async for (verb, n1iden) in src.iterEdgesN2():
739
+
740
+ if verb not in verbs:
741
+ self.runt.layerConfirm(('node', 'edge', 'add', verb))
742
+ verbs.add(verb)
743
+
744
+ n1proto = await editor.getNodeByBuid(s_common.uhex(n1iden))
745
+ if n1proto is not None:
746
+ await n1proto.addEdge(verb, dstiden)
747
+
748
+ async def copyTags(self, src, proto, overwrite=False):
749
+
750
+ for name, valu in src.tags.items():
751
+ self.runt.layerConfirm(('node', 'tag', 'add', *name.split('.')))
752
+ await proto.addTag(name, valu=valu)
753
+
754
+ for tagname, tagprops in src.tagprops.items():
755
+ for propname, valu in tagprops.items():
756
+ if overwrite or not proto.hasTagProp(tagname, propname):
757
+ await proto.setTagProp(tagname, propname, valu) # use tag perms
758
+
759
+
760
+ @s_stormtypes.registry.registerLib
761
+ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
762
+ '''
763
+ A Storm library containing migration tools.
764
+ '''
765
+ _storm_locals = (
766
+ {'name': 'copyData', 'desc': 'Copy node data from the src node to the dst node.',
767
+ 'type': {'type': 'function', '_funcname': '_methCopyData',
768
+ 'args': (
769
+ {'name': 'src', 'type': 'node', 'desc': 'The node to copy data from.', },
770
+ {'name': 'dst', 'type': 'node', 'desc': 'The node to copy data to.', },
771
+ {'name': 'overwrite', 'type': 'boolean', 'default': False,
772
+ 'desc': 'Copy data even if the key exists on the destination node.', },
773
+ ),
774
+ 'returns': {'type': 'null', }}},
775
+ {'name': 'copyEdges', 'desc': 'Copy edges from the src node to the dst node.',
776
+ 'type': {'type': 'function', '_funcname': '_methCopyEdges',
777
+ 'args': (
778
+ {'name': 'src', 'type': 'node', 'desc': 'The node to copy edges from.', },
779
+ {'name': 'dst', 'type': 'node', 'desc': 'The node to copy edges to.', },
780
+ ),
781
+ 'returns': {'type': 'null', }}},
782
+ {'name': 'copyTags', 'desc': 'Copy tags, tag timestamps, and tag props from the src node to the dst node.',
783
+ 'type': {'type': 'function', '_funcname': '_methCopyTags',
784
+ 'args': (
785
+ {'name': 'src', 'type': 'node', 'desc': 'The node to copy tags from.', },
786
+ {'name': 'dst', 'type': 'node', 'desc': 'The node to copy tags to.', },
787
+ {'name': 'overwrite', 'type': 'boolean', 'default': False,
788
+ 'desc': 'Copy tag property value even if the property exists on the destination node.', },
789
+ ),
790
+ 'returns': {'type': 'null', }}},
791
+ )
792
+ _storm_lib_path = ('model', 'migration')
793
+
794
+ def getObjLocals(self):
795
+ return {
796
+ 'copyData': self._methCopyData,
797
+ 'copyEdges': self._methCopyEdges,
798
+ 'copyTags': self._methCopyTags,
799
+ }
800
+
801
+ async def _methCopyData(self, src, dst, overwrite=False):
802
+
803
+ if not isinstance(src, s_node.Node):
804
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyData() source argument must be a node.')
805
+ if not isinstance(dst, s_node.Node):
806
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyData() dest argument must be a node.')
807
+
808
+ overwrite = await s_stormtypes.tobool(overwrite)
809
+
810
+ async with self.runt.snap.getEditor() as editor:
811
+ proto = editor.loadNode(dst)
812
+ await self.copyData(src, proto, overwrite=overwrite)
813
+
814
+ async def _methCopyEdges(self, src, dst):
815
+
816
+ if not isinstance(src, s_node.Node):
817
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyEdges() source argument must be a node.')
818
+ if not isinstance(dst, s_node.Node):
819
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyEdges() dest argument must be a node.')
820
+
821
+ snap = self.runt.snap
822
+
823
+ async with snap.getEditor() as editor:
824
+ proto = editor.loadNode(dst)
825
+ await self.copyEdges(editor, src, proto)
826
+
827
+ async def _methCopyTags(self, src, dst, overwrite=False):
828
+
829
+ if not isinstance(src, s_node.Node):
830
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyTags() source argument must be a node.')
831
+ if not isinstance(dst, s_node.Node):
832
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyTags() dest argument must be a node.')
833
+
834
+ overwrite = await s_stormtypes.tobool(overwrite)
835
+
836
+ snap = self.runt.snap
837
+
838
+ async with snap.getEditor() as editor:
839
+ proto = editor.loadNode(dst)
840
+ await self.copyTags(src, proto, overwrite=overwrite)
841
+
842
+ @s_stormtypes.registry.registerLib
843
+ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
844
+ '''
845
+ A Storm library for selectively migrating nodes in the current view.
846
+ '''
847
+ _storm_locals = (
848
+ {'name': 'itSecCpe_2_170_0',
849
+ 'desc': '''
850
+ Versions of Synapse prior to v2.169.0 did not correctly parse and
851
+ convert CPE strings from 2.2 -> 2.3 or 2.3 -> 2.2. This migration
852
+ attempts to re-normalize `it:sec:cpe` nodes that may be fixable.
853
+
854
+ NOTE: It is highly recommended to test the `it:sec:cpe` migrations
855
+ in a fork first and confirm the migration was successful without any
856
+ issues. Then run the migration in view deporder to migrate the
857
+ entire cortex. E.g.::
858
+
859
+ for $view in $lib.view.list(deporder=$lib.true) {
860
+ view.exec $view.iden {
861
+ for $n in $lib.layer.get().liftByProp(it:sec:cpe) {
862
+ $lib.model.migration.s.itSecCpe_2_170_0($n)
863
+ }
864
+ }
865
+ }
866
+
867
+ Upon completion of the migration, nodedata will contain a
868
+ `migration.s.itSecCpe_2_170_0` dict with information about the
869
+ migration status. This dict may contain the following:
870
+
871
+ - `status`: (required str) "success" or "failed"
872
+ - `reason`: (optional str) if "status" is "failed", this key will
873
+ explain why the migration failed.
874
+ - `valu`: (optional str) if this key is present, it will contain
875
+ an updated CPE2.3 string since the primary property cannot be
876
+ changed.
877
+ - `updated`: (optional list[str]) A list of properties that were
878
+ updated by the migration.
879
+
880
+ Failed or incorrect migrations may be helped by updating the :v2_2
881
+ property to be a valid CPE2.2 string and then re-running the
882
+ migration with `force=$lib.true`. If the primary property (CPE2.3)
883
+ is valid but incorrect, users may update the :v2_2 property and then
884
+ run the migration with `prefer_v22=$lib.true` to make the migration
885
+ use the `:v2_2` string instead of the primary property for the
886
+ migration process.
887
+ ''',
888
+ 'type': {'type': 'function', '_funcname': '_itSecCpe_2_170_0',
889
+ 'args': (
890
+ {'name': 'n', 'type': 'node', 'desc': 'The it:sec:cpe node to migrate.'},
891
+ {'name': 'prefer_v22', 'type': 'bool', 'default': False,
892
+ 'desc': '''
893
+ Try to renormalize using the :v2_2 prop instead of the
894
+ primary property. This can be especially useful when the
895
+ primary property is a valid but incorrect CPE string.
896
+ '''},
897
+ {'name': 'force', 'type': 'bool', 'default': False,
898
+ 'desc': 'Perform fixups even if the primary property and :v2_2 are valid.'},
899
+ ),
900
+ 'returns': {'type': 'boolean', 'desc': 'Boolean indicating if the migration was successful.'}}},
901
+ {'name': 'riskHasVulnToVulnerable', 'desc': '''
902
+ Create a risk:vulnerable node from the provided risk:hasvuln node.
903
+
904
+ Edits will be made to the risk:vulnerable node in the current write layer.
905
+
906
+ If multiple vulnerable properties are set on the risk:hasvuln node
907
+ multiple risk:vulnerable nodes will be created (each with a unique guid).
908
+ Otherwise, a single risk:vulnerable node will be created with the same guid
909
+ as the provided risk:hasvuln node. Extended properties will not be migrated.
910
+
911
+ Tags, tag properties, edges, and node data will be copied
912
+ to the risk:vulnerable node. However, existing tag properties and
913
+ node data will not be overwritten.
914
+ ''',
915
+ 'type': {'type': 'function', '_funcname': '_riskHasVulnToVulnerable',
916
+ 'args': (
917
+ {'name': 'n', 'type': 'node', 'desc': 'The risk:hasvuln node to migrate.'},
918
+ {'name': 'nodata', 'type': 'bool', 'default': False,
919
+ 'desc': 'Do not copy nodedata to the risk:vulnerable node.'},
920
+ ),
921
+ 'returns': {'type': 'list', 'desc': 'A list of idens for the risk:vulnerable nodes.'}}},
922
+ )
923
+ _storm_lib_path = ('model', 'migration', 's')
924
+
925
+ def getObjLocals(self):
926
+ return {
927
+ 'itSecCpe_2_170_0': self._itSecCpe_2_170_0,
928
+ 'riskHasVulnToVulnerable': self._riskHasVulnToVulnerable,
929
+ }
930
+
931
+ async def _itSecCpe_2_170_0(self, n, prefer_v22=False, force=False):
932
+
933
+ if not isinstance(n, s_node.Node):
934
+ raise s_exc.BadArg(mesg='$lib.model.migration.s.itSecCpe_2_170_0() argument must be a node.')
935
+
936
+ if n.form.name != 'it:sec:cpe':
937
+ raise s_exc.BadArg(f'itSecCpeFix only accepts it:sec:cpe nodes, not {n.form.name}')
938
+
939
+ prefer_v22 = await s_stormtypes.tobool(prefer_v22)
940
+ force = await s_stormtypes.tobool(force)
941
+
942
+ layr = self.runt.snap.wlyr
943
+ # We only need to check :v2_2 since that's the only property that's
944
+ # writable. Everthing else is readonly. And we can do it here once
945
+ # instead of in the loop below which will cause a perf hit.
946
+ self.runt.confirmPropSet(n.form.prop('v2_2'), layriden=layr.iden)
947
+
948
+ curv = n.repr()
949
+ reprvalu = f'it:sec:cpe={curv}'
950
+
951
+ nodedata = await n.getData('migration.s.itSecCpe_2_170_0', {})
952
+ if nodedata.get('status') == 'success' and not force:
953
+ if self.runt.debug:
954
+ mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node already migrated.'
955
+ await self.runt.printf(mesg)
956
+ return True
957
+
958
+ modl = self.runt.model.type('it:sec:cpe')
959
+
960
+ valu23 = None
961
+ valu22 = None
962
+ invalid = ''
963
+
964
+ # Check the primary property for validity.
965
+ cpe23 = s_infotech.cpe23_regex.match(curv)
966
+ if cpe23 is not None and cpe23.group() == curv:
967
+ valu23 = curv
968
+
969
+ # Check the v2_2 property for validity.
970
+ v2_2 = n.props.get('v2_2')
971
+ if v2_2 is not None:
972
+ rgx = s_infotech.cpe22_regex.match(v2_2)
973
+ if rgx is not None and rgx.group() == v2_2:
974
+ valu22 = v2_2
975
+
976
+ async with self.runt.snap.getNodeEditor(n) as proto:
977
+
978
+ # If both values are populated, this node is valid
979
+ if valu23 is not None and valu22 is not None and not force:
980
+ if self.runt.debug:
981
+ mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node is valid, no migration necessary.'
982
+ await self.runt.printf(mesg)
983
+
984
+ await proto.setData('migration.s.itSecCpe_2_170_0', {
985
+ 'status': 'success',
986
+ })
987
+
988
+ return True
989
+
990
+ if valu23 is None and valu22 is None:
991
+ reason = 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.'
992
+ # Invalid 2.3 string and no/invalid v2_2 prop. Nothing
993
+ # we can do here so log, mark, and go around.
994
+ mesg = f'itSecCpe_2_170_0({reprvalu}): {reason}'
995
+ await self.runt.warn(mesg)
996
+
997
+ await proto.setData('migration.s.itSecCpe_2_170_0', {
998
+ 'status': 'failed',
999
+ 'reason': reason,
1000
+ })
1001
+
1002
+ return False
1003
+
1004
+ if prefer_v22:
1005
+ valu = valu22 or valu23
1006
+ else:
1007
+ valu = valu23 or valu22
1008
+
1009
+ # Re-normalize the data from the 2.3 or 2.2 string, whichever was valid.
1010
+ norm, info = modl.norm(valu)
1011
+ subs = info.get('subs')
1012
+
1013
+ edits = []
1014
+ nodedata = {'status': 'success'}
1015
+
1016
+ if norm != curv:
1017
+ # The re-normed value is not the same as the current value.
1018
+ # Since we can't change the primary property, store the
1019
+ # updated value in nodedata.
1020
+ if self.runt.debug:
1021
+ mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Stored updated primary property value to nodedata: {curv} -> {norm}.'
1022
+ await self.runt.printf(mesg)
1023
+
1024
+ nodedata['valu'] = norm
1025
+
1026
+ # Iterate over the existing properties
1027
+ for propname, propcurv in n.props.items():
1028
+ subscurv = subs.get(propname)
1029
+ if subscurv is None:
1030
+ continue
1031
+
1032
+ if propname == 'v2_2' and isinstance(subscurv, list):
1033
+ subscurv = s_infotech.zipCpe22(subscurv)
1034
+
1035
+ # Values are the same, go around
1036
+ if propcurv == subscurv:
1037
+ continue
1038
+
1039
+ nodedata.setdefault('updated', [])
1040
+ nodedata['updated'].append(propname)
1041
+
1042
+ # Update the existing property with the re-normalized property value.
1043
+ await proto.set(propname, subscurv, ignore_ro=True)
1044
+
1045
+ await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
1046
+
1047
+ if self.runt.debug:
1048
+ if nodedata.get('updated'):
1049
+ mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Updated properties: {", ".join(nodedata["updated"])}.'
1050
+ await self.runt.printf(mesg)
1051
+ else:
1052
+ mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): No property updates required.'
1053
+ await self.runt.printf(mesg)
1054
+
1055
+ return True
1056
+
1057
+ async def _riskHasVulnToVulnerable(self, n, nodata=False):
1058
+
1059
+ nodata = await s_stormtypes.tobool(nodata)
1060
+
1061
+ if not isinstance(n, s_node.Node):
1062
+ raise s_exc.BadArg(mesg='$lib.model.migration.s.riskHasVulnToVulnerable() argument must be a node.')
1063
+
1064
+ if n.form.name != 'risk:hasvuln':
1065
+ mesg = f'$lib.model.migration.s.riskHasVulnToVulnerable() only accepts risk:hasvuln nodes, not {n.form.name}'
1066
+ raise s_exc.BadArg(mesg=mesg)
1067
+
1068
+ retidens = []
1069
+
1070
+ if not (vuln := n.get('vuln')):
1071
+ return retidens
1072
+
1073
+ props = {
1074
+ 'vuln': vuln,
1075
+ }
1076
+
1077
+ links = {prop: valu for prop in RISK_HASVULN_VULNPROPS if (valu := n.get(prop)) is not None}
1078
+
1079
+ match len(links):
1080
+ case 0:
1081
+ return retidens
1082
+ case 1:
1083
+ guid = n.ndef[1]
1084
+ case _:
1085
+ guid = None
1086
+
1087
+ riskvuln = self.runt.model.form('risk:vulnerable')
1088
+
1089
+ self.runt.layerConfirm(riskvuln.addperm)
1090
+ self.runt.confirmPropSet(riskvuln.props['vuln'])
1091
+ self.runt.confirmPropSet(riskvuln.props['node'])
1092
+
1093
+ if (seen := n.get('.seen')):
1094
+ self.runt.confirmPropSet(riskvuln.props['.seen'])
1095
+ props['.seen'] = seen
1096
+
1097
+ async with self.runt.snap.getEditor() as editor:
1098
+
1099
+ for prop, valu in links.items():
1100
+
1101
+ pguid = guid if guid is not None else s_common.guid((guid, prop))
1102
+ pprops = props | {'node': (n.form.props[prop].type.name, valu)}
1103
+
1104
+ proto = await editor.addNode('risk:vulnerable', pguid, props=pprops)
1105
+ retidens.append(proto.iden())
1106
+
1107
+ await self.copyTags(n, proto, overwrite=False)
1108
+ await self.copyEdges(editor, n, proto)
1109
+
1110
+ if not nodata:
1111
+ await self.copyData(n, proto, overwrite=False)
1112
+
1113
+ return retidens
synapse/lib/stormtypes.py CHANGED
@@ -1604,8 +1604,12 @@ class LibBase(Lib):
1604
1604
  continue
1605
1605
  vals.append(arg)
1606
1606
 
1607
+ if len(vals) < 1:
1608
+ mesg = '$lib.min() must have at least one argument or a list containing at least one value.'
1609
+ raise s_exc.StormRuntimeError(mesg=mesg)
1610
+
1607
1611
  ints = [await toint(x) for x in vals]
1608
- return min(*ints)
1612
+ return min(ints)
1609
1613
 
1610
1614
  @stormfunc(readonly=True)
1611
1615
  async def _max(self, *args):
@@ -1618,8 +1622,12 @@ class LibBase(Lib):
1618
1622
  continue
1619
1623
  vals.append(arg)
1620
1624
 
1625
+ if len(vals) < 1:
1626
+ mesg = '$lib.max() must have at least one argument or a list containing at least one value.'
1627
+ raise s_exc.StormRuntimeError(mesg=mesg)
1628
+
1621
1629
  ints = [await toint(x) for x in vals]
1622
- return max(*ints)
1630
+ return max(ints)
1623
1631
 
1624
1632
  @staticmethod
1625
1633
  async def _get_mesg(mesg, **kwargs):
@@ -6713,6 +6721,20 @@ class Layer(Prim):
6713
6721
  ''',
6714
6722
  'type': {'type': 'function', '_funcname': 'getStorNodes',
6715
6723
  'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Tuple of buid, sode values.', }}},
6724
+ {'name': 'getStorNodesByForm', 'desc': '''
6725
+ Get buid, sode tuples representing the data stored in the layer for a given form.
6726
+
6727
+ Notes:
6728
+ The storage nodes represent **only** the data stored in the layer
6729
+ and may not represent whole nodes. If the only data stored in the layer for
6730
+ a given buid is an N2 edge reference, a storage node will not be returned.
6731
+ ''',
6732
+ 'type': {'type': 'function', '_funcname': 'getStorNodesByForm',
6733
+ 'args': (
6734
+ {'name': 'form', 'type': 'str',
6735
+ 'desc': 'The name of the form to get storage nodes for.'},
6736
+ ),
6737
+ 'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Tuple of buid, sode values.', }}},
6716
6738
  {'name': 'getMirrorStatus', 'desc': '''
6717
6739
  Return a dictionary of the mirror synchronization status for the layer.
6718
6740
  ''',
@@ -6894,6 +6916,7 @@ class Layer(Prim):
6894
6916
  'getFormCounts': self._methGetFormcount,
6895
6917
  'getStorNode': self.getStorNode,
6896
6918
  'getStorNodes': self.getStorNodes,
6919
+ 'getStorNodesByForm': self.getStorNodesByForm,
6897
6920
  'getEdgesByN1': self.getEdgesByN1,
6898
6921
  'getEdgesByN2': self.getEdgesByN2,
6899
6922
  'getMirrorStatus': self.getMirrorStatus,
@@ -7190,6 +7213,19 @@ class Layer(Prim):
7190
7213
  async for item in layr.getStorNodes():
7191
7214
  yield item
7192
7215
 
7216
+ @stormfunc(readonly=True)
7217
+ async def getStorNodesByForm(self, form):
7218
+ form = await tostr(form)
7219
+ if self.runt.snap.core.model.form(form) is None:
7220
+ raise s_exc.NoSuchForm.init(form)
7221
+
7222
+ layriden = self.valu.get('iden')
7223
+ await self.runt.reqUserCanReadLayer(layriden)
7224
+ layr = self.runt.snap.core.getLayer(layriden)
7225
+
7226
+ async for item in layr.getStorNodesByForm(form):
7227
+ yield item
7228
+
7193
7229
  @stormfunc(readonly=True)
7194
7230
  async def getEdges(self):
7195
7231
  layriden = self.valu.get('iden')
@@ -7444,6 +7480,12 @@ class View(Prim):
7444
7480
  {'name': 'name', 'type': 'str', 'desc': 'The name of the new view.', 'default': None, },
7445
7481
  ),
7446
7482
  'returns': {'type': 'view', 'desc': 'The ``view`` object for the new View.', }}},
7483
+ {'name': 'insertParentFork', 'desc': 'Insert a new View between a forked View and its parent.',
7484
+ 'type': {'type': 'function', '_funcname': '_methViewInsertParentFork',
7485
+ 'args': (
7486
+ {'name': 'name', 'type': 'str', 'desc': 'The name of the new View.', 'default': None},
7487
+ ),
7488
+ 'returns': {'type': 'view', 'desc': 'The ``view`` object for the new View.', }}},
7447
7489
  {'name': 'pack', 'desc': 'Get the View definition.',
7448
7490
  'type': {'type': 'function', '_funcname': '_methViewPack',
7449
7491
  'returns': {'type': 'dict', 'desc': 'Dictionary containing the View definition.', }}},
@@ -7645,7 +7687,6 @@ class View(Prim):
7645
7687
  return {
7646
7688
  'set': self._methViewSet,
7647
7689
  'get': self._methViewGet,
7648
- 'fork': self._methViewFork,
7649
7690
  'pack': self._methViewPack,
7650
7691
  'repr': self._methViewRepr,
7651
7692
  'merge': self._methViewMerge,
@@ -7660,6 +7701,9 @@ class View(Prim):
7660
7701
  'getTagPropCount': self._methGetTagPropCount,
7661
7702
  'getPropArrayCount': self._methGetPropArrayCount,
7662
7703
 
7704
+ 'fork': self._methViewFork,
7705
+ 'insertParentFork': self._methViewInsertParentFork,
7706
+
7663
7707
  'getMerges': self.getMerges,
7664
7708
  'delMergeVote': self.delMergeVote,
7665
7709
  'setMergeVote': self.setMergeVote,
@@ -7911,6 +7955,27 @@ class View(Prim):
7911
7955
 
7912
7956
  return View(self.runt, newv, path=self.path)
7913
7957
 
7958
+ async def _methViewInsertParentFork(self, name=None):
7959
+ useriden = self.runt.user.iden
7960
+ viewiden = self.valu.get('iden')
7961
+
7962
+ name = await tostr(name, noneok=True)
7963
+
7964
+ self.runt.reqAdmin(gateiden=viewiden)
7965
+
7966
+ view = self.runt.snap.core.reqView(viewiden)
7967
+ if not view.isafork():
7968
+ mesg = f'View ({viewiden}) is not a fork, cannot insert a new fork between it and parent.'
7969
+ raise s_exc.BadState(mesg=mesg)
7970
+
7971
+ self.runt.confirm(('view', 'add'))
7972
+ self.runt.confirm(('view', 'read'), gateiden=view.parent.iden)
7973
+ self.runt.confirm(('view', 'fork'), gateiden=view.parent.iden)
7974
+
7975
+ newv = await view.insertParentFork(useriden, name=name)
7976
+
7977
+ return View(self.runt, newv, path=self.path)
7978
+
7914
7979
  async def _methViewMerge(self, force=False):
7915
7980
  '''
7916
7981
  Merge a forked view back into its parent.
synapse/lib/types.py CHANGED
@@ -1408,6 +1408,31 @@ class Ndef(Type):
1408
1408
  self.setNormFunc(list, self._normPyTuple)
1409
1409
  self.setNormFunc(tuple, self._normPyTuple)
1410
1410
 
1411
+ self.formfilter = None
1412
+
1413
+ self.forms = self.opts.get('forms')
1414
+ self.ifaces = self.opts.get('interfaces')
1415
+
1416
+ if self.forms or self.ifaces:
1417
+ if self.forms is not None:
1418
+ forms = set(self.forms)
1419
+
1420
+ if self.ifaces is not None:
1421
+ ifaces = set(self.ifaces)
1422
+
1423
+ def filtfunc(form):
1424
+ if self.forms is not None and form.name in forms:
1425
+ return False
1426
+
1427
+ if self.ifaces is not None:
1428
+ for iface in form.ifaces.keys():
1429
+ if iface in ifaces:
1430
+ return False
1431
+
1432
+ return True
1433
+
1434
+ self.formfilter = filtfunc
1435
+
1411
1436
  def _normStormNode(self, valu):
1412
1437
  return self._normPyTuple(valu.ndef)
1413
1438
 
@@ -1421,6 +1446,16 @@ class Ndef(Type):
1421
1446
  if form is None:
1422
1447
  raise s_exc.NoSuchForm.init(formname)
1423
1448
 
1449
+ if self.formfilter is not None and self.formfilter(form):
1450
+ mesg = f'Ndef of form {formname} is not allowed as a value for {self.name} with form filter'
1451
+ if self.forms is not None:
1452
+ mesg += f' forms={self.forms}'
1453
+
1454
+ if self.ifaces is not None:
1455
+ mesg += f' interfaces={self.ifaces}'
1456
+
1457
+ raise s_exc.BadTypeValu(valu=formname, name=self.name, mesg=mesg, forms=self.forms, interfaces=self.ifaces)
1458
+
1424
1459
  formnorm, forminfo = form.type.norm(formvalu)
1425
1460
  norm = (form.name, formnorm)
1426
1461
 
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 168, 0)
226
+ version = (2, 170, 0)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = 'd07eb2560a6e5e025ce9c20efaafe5571a55856c'
228
+ commit = '0e74d28df290e631963a8e06c8e605531a96aeab'