synapse 2.201.0__py311-none-any.whl → 2.203.0__py311-none-any.whl

synapse/tests/test_lib_grammar.py

@@ -192,17 +192,17 @@ Queries = [
     '[ test:str=woot .seen=(2014,2015) ]',
     '[ test:str=woot .seen=20 ]',
     '[-#foo]',
-    '[meta:seen=((test:str, foobar), (test:str, foo))]',
-    '[meta:seen=((test:comp, (2048, horton)), (test:comp, (4096, whoville)))]',
-    '[meta:seen=((test:comp, (9001, "A mean one")), (test:comp, (40000, greeneggs)))]',
-    '[meta:seen=((test:int, 16), (test:comp, (9999, greenham)))]',
-    '[meta:seen=((test:str, 123), (test:int, 123))]',
+    '[meta:source=((test:str, foobar), (test:str, foo))]',
+    '[meta:source=((test:comp, (2048, horton)), (test:comp, (4096, whoville)))]',
+    '[meta:source=((test:comp, (9001, "A mean one")), (test:comp, (40000, greeneggs)))]',
+    '[meta:source=((test:int, 16), (test:comp, (9999, greenham)))]',
+    '[meta:source=((test:str, 123), (test:int, 123))]',
     '[inet:dns:query=(tcp://1.2.3.4, "", 1)]',
     '[inet:dns:query=(tcp://1.2.3.4, "foo*.haha.com", 1)]',
     '[inet:ipv4=1.2.3.1-1.2.3.3]',
-    '[inet:ipv4=1.2.3.4 :asn=10] [meta:seen=(abcd, (inet:asn, 10))]',
-    '[meta:seen=(abcd, (test:str, pennywise))]',
-    '[meta:source=abcd +#omit.nopiv] [meta:seen=(abcd, (test:pivtarg, foo))]',
+    '[inet:ipv4=1.2.3.4 :asn=10] [meta:source=(abcd, (inet:asn, 10))]',
+    '[meta:source=(abcd, (test:str, pennywise))]',
+    '[meta:source=abcd +#omit.nopiv] [meta:source=(abcd, (test:pivtarg, foo))]',
     '[test:comp=(1234, 5678)]',
     '[test:comp=(3, foob) +#meep.gorp +#bleep.zlorp +#cond]',
     '[test:guid="*" :tick=2001]',
@@ -220,8 +220,8 @@ Queries = [
     '[test:str=foo :tick=201808021201]',
     '[test:str=hehe] | iden abcd | count',
     '[test:str=hello]',
-    'meta:seen +:node*range=((test:comp, (1000, green)), (test:comp, (3000, ham)))',
-    'meta:seen',
+    'meta:source +:node*range=((test:comp, (1000, green)), (test:comp, (3000, ham)))',
+    'meta:source',
     'file:bytes:size=4',
     'for $fqdn in $fqdns { [ inet:fqdn=$fqdn ] }',
     'for ($fqdn, $ipv4) in $dnsa { [ inet:dns:a=($fqdn,$ipv4) ] }',
@@ -232,8 +232,8 @@ Queries = [
     'geo:place:latlong*near=(("34.118560", "-118.300370"), 50m)',
     'geo:place:latlong*near=((0, 0), 50m)',
     'geo:place:latlong*near=((34.1, -118.3), 10km)',
-    'geo:place=$place <- meta:seen <- *',
-    'geo:place=$place <- meta:seen <- ps:person',
+    'geo:place=$place <- meta:source <- *',
+    'geo:place=$place <- meta:source <- ps:person',
     'geo:place=abcd $latlong=:latlong $radius=:radius | spin | tel:mob:telem:latlong*near=($latlong, 3km)',
     'meta:note=abcd | noderefs -d 2 --join',
     'help',
@@ -285,16 +285,16 @@ Queries = [
     'inet:user | limit 10 | +inet:user=visi',
     'inet:user | limit 10 | [ +#foo.bar ]',
     'media:news = 00a1f0d928e25729b9e86e2d08c127ce [ :summary = \"\" ]',
-    'meta:seen:meta:source=$sorc -> *',
-    'meta:seen:meta:source=$sorc :node -> *',
+    'meta:source:meta:source=$sorc -> *',
+    'meta:source:meta:source=$sorc :node -> *',
     'meta:source=8f1401de15918358d5247e21ca29a814',
     'movetag a.b a.m',
     'movetag hehe woot',
-    'ps:person=$pers -> meta:seen -> *',
-    'ps:person=$pers -> meta:seen -> geo:place',
-    'ps:person=$pers -> meta:seen +:time@=(2014,2017) -> geo:place',
-    'ps:person=$pers -> meta:seen -> *',
-    'ps:person=$pers -> meta:seen :node -> *',
+    'ps:person=$pers -> meta:source -> *',
+    'ps:person=$pers -> meta:source -> geo:place',
+    'ps:person=$pers -> meta:source +:time@=(2014,2017) -> geo:place',
+    'ps:person=$pers -> meta:source -> *',
+    'ps:person=$pers -> meta:source :node -> *',
     'reindex --form-counts',
     'sudo | [ inet:ipv4=1.2.3.4 ]',
     'sudo | [ test:cycle0=foo :test:cycle1=bar ]',
@@ -444,12 +444,12 @@ Queries = [
     'test:str=bar <- *',
     'test:str=bar test:pivcomp=(foo,bar) [+#test.bar]',
     'test:str=foo +#lol@=2016',
-    'test:str=foo <+- meta:seen',
-    'test:str=foo <- meta:seen',
+    'test:str=foo <+- meta:source',
+    'test:str=foo <- meta:source',
     'test:str=foo | delnode',
-    'test:str=foobar -+> meta:seen',
-    'test:str=foobar -> meta:seen <+- test:str',
-    'test:str=foobar -> meta:seen <- test:str',
+    'test:str=foobar -+> meta:source',
+    'test:str=foobar -> meta:source <+- test:str',
+    'test:str=foobar -> meta:source <- test:str',
     'test:str=hello [:tick="2001"]',
     'test:str=hello [:tick="2002"]',
     'test:str=pennywise | noderefs --join -d 9 --traverse-edge',
@@ -476,7 +476,7 @@ Queries = [
         inet:fqdn | graph
                     --degrees 2
                     --filter { -#nope }
-                    --pivot { <- meta:seen <- meta:source }
+                    --pivot { <- meta:source <- meta:source }
                     --form-pivot inet:fqdn {<- * | limit 20}
                     --form-pivot inet:fqdn {-> * | limit 20}
                     --form-filter inet:fqdn {-inet:fqdn:issuffix=1}
@@ -514,7 +514,7 @@ Queries = [
 
     {[ inet:email:message:attachment=($node, "*") ] -inet:email:message [ :name=sploit.exe ]}
 
-    {[ meta:seen=($node, ('inet:email:header', ('to', 'Visi Kensho <visi@vertex.link>'))) ]}
+    {[ meta:source=($node, ('inet:email:header', ('to', 'Visi Kensho <visi@vertex.link>'))) ]}
     ''',
     '$x = $(1 / 3)',
     '$x = $(1 * 3)',
@@ -690,18 +690,18 @@ Queries = [
     'reverse(*$form=$valu)',
     'test:str=foobar -> inet:dns*',
     'test:str=foobar -> inet:dns:*',
-    'test:str=foobar -> (meta:seen, inet:dns:a)',
-    'test:str=foobar -> (meta:seen, inet:dns*)',
+    'test:str=foobar -> (meta:source, inet:dns:a)',
+    'test:str=foobar -> (meta:source, inet:dns*)',
     'test:str=foobar -> $foo',
     'test:str=foobar -+> inet:dns*',
     'test:str=foobar -+> inet:dns:*',
-    'test:str=foobar -+> (meta:seen, inet:dns:a)',
-    'test:str=foobar -+> (meta:seen, inet:dns*)',
+    'test:str=foobar -+> (meta:source, inet:dns:a)',
+    'test:str=foobar -+> (meta:source, inet:dns*)',
     'test:str=foobar -+> $foo',
     'test:str=foobar -(refs)> inet:dns:*',
-    'inet:fqdn=foo.com :zone -> (meta:seen, inet:dns:a)',
+    'inet:fqdn=foo.com :zone -> (meta:source, inet:dns:a)',
     'inet:fqdn=foo.com :zone -> $foo',
-    'inet:fqdn=foo.com :zone -+> (meta:seen, inet:dns:a)',
+    'inet:fqdn=foo.com :zone -+> (meta:source, inet:dns:a)',
     'inet:fqdn=foo.com :zone -+> $foo',
     'test:*#foo',
     'test:*#foo@=2016',
@@ -746,6 +746,8 @@ Queries = [
     '[test:str=foo :bar--=(foo, bar)]',
     '[test:str=foo :bar?++=$baz]',
     '[test:str=foo :bar?--={[it:dev:str=foo]}]',
+    '$foo=(notime,)',
+    '$foo=(nulltime,)',
 ]
 
 # Generated with print_parse_list below
@@ -931,17 +933,17 @@ _ParseResults = [
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: woot], EditPropSet: [UnivProp: [Const: .seen], Const: =, List: [Const: 2014, Const: 2015]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: woot], EditPropSet: [UnivProp: [Const: .seen], Const: =, Const: 20]]',
     'Query: [EditTagDel: [TagName: [Const: foo]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [List: [Const: test:str, Const: foobar], List: [Const: test:str, Const: foo]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [List: [Const: test:comp, List: [Const: 2048, Const: horton]], List: [Const: test:comp, List: [Const: 4096, Const: whoville]]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [List: [Const: test:comp, List: [Const: 9001, Const: A mean one]], List: [Const: test:comp, List: [Const: 40000, Const: greeneggs]]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [List: [Const: test:int, Const: 16], List: [Const: test:comp, List: [Const: 9999, Const: greenham]]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [List: [Const: test:str, Const: 123], List: [Const: test:int, Const: 123]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [List: [Const: test:str, Const: foobar], List: [Const: test:str, Const: foo]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [List: [Const: test:comp, List: [Const: 2048, Const: horton]], List: [Const: test:comp, List: [Const: 4096, Const: whoville]]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [List: [Const: test:comp, List: [Const: 9001, Const: A mean one]], List: [Const: test:comp, List: [Const: 40000, Const: greeneggs]]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [List: [Const: test:int, Const: 16], List: [Const: test:comp, List: [Const: 9999, Const: greenham]]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [List: [Const: test:str, Const: 123], List: [Const: test:int, Const: 123]]]]',
     'Query: [EditNodeAdd: [FormName: [Const: inet:dns:query], Const: =, List: [Const: tcp://1.2.3.4, Const: , Const: 1]]]',
     'Query: [EditNodeAdd: [FormName: [Const: inet:dns:query], Const: =, List: [Const: tcp://1.2.3.4, Const: foo*.haha.com, Const: 1]]]',
     'Query: [EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.1-1.2.3.3]]',
-    'Query: [EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4], EditPropSet: [RelProp: [Const: asn], Const: =, Const: 10], EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [Const: abcd, List: [Const: inet:asn, Const: 10]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [Const: abcd, List: [Const: test:str, Const: pennywise]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, Const: abcd], EditTagAdd: [TagName: [Const: omit, Const: nopiv]], EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [Const: abcd, List: [Const: test:pivtarg, Const: foo]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4], EditPropSet: [RelProp: [Const: asn], Const: =, Const: 10], EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [Const: abcd, List: [Const: inet:asn, Const: 10]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [Const: abcd, List: [Const: test:str, Const: pennywise]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, Const: abcd], EditTagAdd: [TagName: [Const: omit, Const: nopiv]], EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [Const: abcd, List: [Const: test:pivtarg, Const: foo]]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:comp], Const: =, List: [Const: 1234, Const: 5678]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:comp], Const: =, List: [Const: 3, Const: foob]], EditTagAdd: [TagName: [Const: meep, Const: gorp]], EditTagAdd: [TagName: [Const: bleep, Const: zlorp]], EditTagAdd: [TagName: [Const: cond]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:guid], Const: =, Const: *], EditPropSet: [RelProp: [Const: tick], Const: =, Const: 2001]]',
@@ -959,8 +961,8 @@ _ParseResults = [
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSet: [RelProp: [Const: tick], Const: =, Const: 201808021201]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: hehe], CmdOper: [Const: iden, List: [Const: abcd]], CmdOper: [Const: count, Const: ()]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: hello]]',
-    'Query: [LiftProp: [Const: meta:seen], FiltOper: [Const: +, RelPropCond: [RelPropValue: [RelProp: [Const: node]], Const: range=, List: [List: [Const: test:comp, List: [Const: 1000, Const: green]], List: [Const: test:comp, List: [Const: 3000, Const: ham]]]]]]',
-    'Query: [LiftProp: [Const: meta:seen]]',
+    'Query: [LiftProp: [Const: meta:source], FiltOper: [Const: +, RelPropCond: [RelPropValue: [RelProp: [Const: node]], Const: range=, List: [List: [Const: test:comp, List: [Const: 1000, Const: green]], List: [Const: test:comp, List: [Const: 3000, Const: ham]]]]]]',
+    'Query: [LiftProp: [Const: meta:source]]',
     'Query: [LiftPropBy: [Const: file:bytes:size, Const: =, Const: 4]]',
     'Query: [ForLoop: [Const: fqdn, VarValue: [Const: fqdns], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:fqdn], Const: =, VarValue: [Const: fqdn]]]]]]',
     "Query: [ForLoop: [VarList: ['fqdn', 'ipv4'], VarValue: [Const: dnsa], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:dns:a], Const: =, List: [VarValue: [Const: fqdn], VarValue: [Const: ipv4]]]]]]]",
@@ -971,8 +973,8 @@ _ParseResults = [
     'Query: [LiftPropBy: [Const: geo:place:latlong, Const: near=, List: [List: [Const: 34.118560, Const: -118.300370], Const: 50m]]]',
     'Query: [LiftPropBy: [Const: geo:place:latlong, Const: near=, List: [List: [Const: 0, Const: 0], Const: 50m]]]',
     'Query: [LiftPropBy: [Const: geo:place:latlong, Const: near=, List: [List: [Const: 34.1, Const: -118.3], Const: 10km]]]',
-    'Query: [LiftPropBy: [Const: geo:place, Const: =, VarValue: [Const: place]], PivotInFrom: [Const: meta:seen], isjoin=False, PivotIn: [], isjoin=False]',
-    'Query: [LiftPropBy: [Const: geo:place, Const: =, VarValue: [Const: place]], PivotInFrom: [Const: meta:seen], isjoin=False, PivotInFrom: [Const: ps:person], isjoin=False]',
+    'Query: [LiftPropBy: [Const: geo:place, Const: =, VarValue: [Const: place]], PivotInFrom: [Const: meta:source], isjoin=False, PivotIn: [], isjoin=False]',
+    'Query: [LiftPropBy: [Const: geo:place, Const: =, VarValue: [Const: place]], PivotInFrom: [Const: meta:source], isjoin=False, PivotInFrom: [Const: ps:person], isjoin=False]',
     'Query: [LiftPropBy: [Const: geo:place, Const: =, Const: abcd], SetVarOper: [Const: latlong, RelPropValue: [Const: latlong]], SetVarOper: [Const: radius, RelPropValue: [Const: radius]], CmdOper: [Const: spin, Const: ()], LiftPropBy: [Const: tel:mob:telem:latlong, Const: near=, List: [VarValue: [Const: latlong], Const: 3km]]]',
     'Query: [LiftPropBy: [Const: meta:note, Const: =, Const: abcd], CmdOper: [Const: noderefs, List: [Const: -d, Const: 2, Const: --join]]]',
     'Query: [CmdOper: [Const: help, Const: ()]]',
@@ -1024,16 +1026,16 @@ _ParseResults = [
     'Query: [LiftProp: [Const: inet:user], CmdOper: [Const: limit, List: [Const: 10]], FiltOper: [Const: +, AbsPropCond: [Const: inet:user, Const: =, Const: visi]]]',
     'Query: [LiftProp: [Const: inet:user], CmdOper: [Const: limit, List: [Const: 10]], EditTagAdd: [TagName: [Const: foo, Const: bar]]]',
     'Query: [LiftPropBy: [Const: media:news, Const: =, Const: 00a1f0d928e25729b9e86e2d08c127ce], EditPropSet: [RelProp: [Const: summary], Const: =, Const: ]]',
-    'Query: [LiftPropBy: [Const: meta:seen:meta:source, Const: =, VarValue: [Const: sorc]], PivotOut: [], isjoin=False]',
-    'Query: [LiftPropBy: [Const: meta:seen:meta:source, Const: =, VarValue: [Const: sorc]], PropPivotOut: [RelProp: [Const: node]], isjoin=False]',
+    'Query: [LiftPropBy: [Const: meta:source:meta:source, Const: =, VarValue: [Const: sorc]], PivotOut: [], isjoin=False]',
+    'Query: [LiftPropBy: [Const: meta:source:meta:source, Const: =, VarValue: [Const: sorc]], PropPivotOut: [RelProp: [Const: node]], isjoin=False]',
     'Query: [LiftPropBy: [Const: meta:source, Const: =, Const: 8f1401de15918358d5247e21ca29a814]]',
     'Query: [CmdOper: [Const: movetag, List: [Const: a.b, Const: a.m]]]',
     'Query: [CmdOper: [Const: movetag, List: [Const: hehe, Const: woot]]]',
-    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:seen], isjoin=False, PivotOut: [], isjoin=False]',
-    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:seen], isjoin=False, FormPivot: [Const: geo:place], isjoin=False]',
-    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:seen], isjoin=False, FiltOper: [Const: +, RelPropCond: [RelPropValue: [RelProp: [Const: time]], Const: @=, List: [Const: 2014, Const: 2017]]], FormPivot: [Const: geo:place], isjoin=False]',
-    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:seen], isjoin=False, PivotOut: [], isjoin=False]',
-    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:seen], isjoin=False, PropPivotOut: [RelProp: [Const: node]], isjoin=False]',
+    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:source], isjoin=False, PivotOut: [], isjoin=False]',
+    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:source], isjoin=False, FormPivot: [Const: geo:place], isjoin=False]',
+    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:source], isjoin=False, FiltOper: [Const: +, RelPropCond: [RelPropValue: [RelProp: [Const: time]], Const: @=, List: [Const: 2014, Const: 2017]]], FormPivot: [Const: geo:place], isjoin=False]',
+    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:source], isjoin=False, PivotOut: [], isjoin=False]',
+    'Query: [LiftPropBy: [Const: ps:person, Const: =, VarValue: [Const: pers]], FormPivot: [Const: meta:source], isjoin=False, PropPivotOut: [RelProp: [Const: node]], isjoin=False]',
     'Query: [CmdOper: [Const: reindex, List: [Const: --form-counts]]]',
     'Query: [CmdOper: [Const: sudo, Const: ()], EditNodeAdd: [FormName: [Const: inet:ipv4], Const: =, Const: 1.2.3.4]]',
     'Query: [CmdOper: [Const: sudo, Const: ()], EditNodeAdd: [FormName: [Const: test:cycle0], Const: =, Const: foo], EditPropSet: [RelProp: [Const: test:cycle1], Const: =, Const: bar]]',
@@ -1183,12 +1185,12 @@ _ParseResults = [
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: bar], PivotIn: [], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: bar], LiftPropBy: [Const: test:pivcomp, Const: =, List: [Const: foo, Const: bar]], EditTagAdd: [TagName: [Const: test, Const: bar]]]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foo], FiltOper: [Const: +, TagValuCond: [TagMatch: [Const: lol], Const: @=, Const: 2016]]]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foo], PivotInFrom: [Const: meta:seen], isjoin=True]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foo], PivotInFrom: [Const: meta:seen], isjoin=False]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foo], PivotInFrom: [Const: meta:source], isjoin=True]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foo], PivotInFrom: [Const: meta:source], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foo], CmdOper: [Const: delnode, Const: ()]]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: meta:seen], isjoin=True]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: meta:seen], isjoin=False, PivotInFrom: [Const: test:str], isjoin=True]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: meta:seen], isjoin=False, PivotInFrom: [Const: test:str], isjoin=False]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: meta:source], isjoin=True]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: meta:source], isjoin=False, PivotInFrom: [Const: test:str], isjoin=True]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: meta:source], isjoin=False, PivotInFrom: [Const: test:str], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: hello], EditPropSet: [RelProp: [Const: tick], Const: =, Const: 2001]]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: hello], EditPropSet: [RelProp: [Const: tick], Const: =, Const: 2002]]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: pennywise], CmdOper: [Const: noderefs, List: [Const: --join, Const: -d, Const: 9, Const: --traverse-edge]]]',
@@ -1204,11 +1206,11 @@ _ParseResults = [
     'Query: [LiftProp: [Const: test:int]]',
     'Query: [LiftProp: [Const: test:int]]',
     'Query: [LiftProp: [Const: test:int]]',
-    'Query: [LiftProp: [Const: inet:fqdn], CmdOper: [Const: graph, List: [Const: --degrees, Const: 2, Const: --filter, ArgvQuery: [Query: [FiltOper: [Const: -, TagCond: [TagMatch: [Const: nope]]]]], Const: --pivot, ArgvQuery: [Query: [PivotInFrom: [Const: meta:seen], isjoin=False, PivotInFrom: [Const: meta:source], isjoin=False]], Const: --form-pivot, Const: inet:fqdn, ArgvQuery: [Query: [PivotIn: [], isjoin=False, CmdOper: [Const: limit, List: [Const: 20]]]], Const: --form-pivot, Const: inet:fqdn, ArgvQuery: [Query: [PivotOut: [], isjoin=False, CmdOper: [Const: limit, List: [Const: 20]]]], Const: --form-filter, Const: inet:fqdn, ArgvQuery: [Query: [FiltOper: [Const: -, AbsPropCond: [Const: inet:fqdn:issuffix, Const: =, Const: 1]]]], Const: --form-pivot, Const: syn:tag, ArgvQuery: [Query: [PivotOut: [], isjoin=False]], Const: --form-pivot, Const: *, ArgvQuery: [Query: [PivotToTags: [TagMatch: []], isjoin=False]]]]]',
+    'Query: [LiftProp: [Const: inet:fqdn], CmdOper: [Const: graph, List: [Const: --degrees, Const: 2, Const: --filter, ArgvQuery: [Query: [FiltOper: [Const: -, TagCond: [TagMatch: [Const: nope]]]]], Const: --pivot, ArgvQuery: [Query: [PivotInFrom: [Const: meta:source], isjoin=False, PivotInFrom: [Const: meta:source], isjoin=False]], Const: --form-pivot, Const: inet:fqdn, ArgvQuery: [Query: [PivotIn: [], isjoin=False, CmdOper: [Const: limit, List: [Const: 20]]]], Const: --form-pivot, Const: inet:fqdn, ArgvQuery: [Query: [PivotOut: [], isjoin=False, CmdOper: [Const: limit, List: [Const: 20]]]], Const: --form-filter, Const: inet:fqdn, ArgvQuery: [Query: [FiltOper: [Const: -, AbsPropCond: [Const: inet:fqdn:issuffix, Const: =, Const: 1]]]], Const: --form-pivot, Const: syn:tag, ArgvQuery: [Query: [PivotOut: [], isjoin=False]], Const: --form-pivot, Const: *, ArgvQuery: [Query: [PivotToTags: [TagMatch: []], isjoin=False]]]]]',
     "Query: [ForLoop: [Const: foo, VarValue: [Const: foos], SubQuery: [Query: [VarListSetOper: [VarList: ['fqdn', 'ipv4'], FuncCall: [VarDeref: [VarValue: [Const: foo], Const: split], CallArgs: [Const: |], CallKwargs: []]], EditNodeAdd: [FormName: [Const: inet:dns:a], Const: =, List: [VarValue: [Const: fqdn], VarValue: [Const: ipv4]]]]]]]",
     'Query: [ForLoop: [Const: tag, FuncCall: [VarDeref: [VarValue: [Const: node], Const: tags], CallArgs: [], CallKwargs: []], SubQuery: [Query: [FormPivot: [Const: test:int], isjoin=False, EditTagAdd: [TagName: [VarValue: [Const: tag]]]]]]]',
     'Query: [ForLoop: [Const: tag, FuncCall: [VarDeref: [VarValue: [Const: node], Const: tags], CallArgs: [Const: fo*], CallKwargs: []], SubQuery: [Query: [FormPivot: [Const: test:int], isjoin=False, EditTagDel: [TagName: [VarValue: [Const: tag]]]]]]]',
-    'Query: [EditNodeAdd: [FormName: [Const: inet:email:message], Const: =, Const: *], EditPropSet: [RelProp: [Const: to], Const: =, Const: woot@woot.com], EditPropSet: [RelProp: [Const: from], Const: =, Const: visi@vertex.link], EditPropSet: [RelProp: [Const: replyto], Const: =, Const: root@root.com], EditPropSet: [RelProp: [Const: subject], Const: =, Const: hi there], EditPropSet: [RelProp: [Const: date], Const: =, Const: 2015], EditPropSet: [RelProp: [Const: body], Const: =, Const: there are mad sploitz here!], EditPropSet: [RelProp: [Const: bytes], Const: =, Const: *], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:email:message:link], Const: =, List: [VarValue: [Const: node], Const: https://www.vertex.link]]]], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:email:message:attachment], Const: =, List: [VarValue: [Const: node], Const: *]], FiltOper: [Const: -, HasAbsPropCond: [Const: inet:email:message]], EditPropSet: [RelProp: [Const: name], Const: =, Const: sploit.exe]]], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: meta:seen], Const: =, List: [VarValue: [Const: node], List: [Const: inet:email:header, List: [Const: to, Const: Visi Kensho <visi@vertex.link>]]]]]]]',
+    'Query: [EditNodeAdd: [FormName: [Const: inet:email:message], Const: =, Const: *], EditPropSet: [RelProp: [Const: to], Const: =, Const: woot@woot.com], EditPropSet: [RelProp: [Const: from], Const: =, Const: visi@vertex.link], EditPropSet: [RelProp: [Const: replyto], Const: =, Const: root@root.com], EditPropSet: [RelProp: [Const: subject], Const: =, Const: hi there], EditPropSet: [RelProp: [Const: date], Const: =, Const: 2015], EditPropSet: [RelProp: [Const: body], Const: =, Const: there are mad sploitz here!], EditPropSet: [RelProp: [Const: bytes], Const: =, Const: *], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:email:message:link], Const: =, List: [VarValue: [Const: node], Const: https://www.vertex.link]]]], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: inet:email:message:attachment], Const: =, List: [VarValue: [Const: node], Const: *]], FiltOper: [Const: -, HasAbsPropCond: [Const: inet:email:message]], EditPropSet: [RelProp: [Const: name], Const: =, Const: sploit.exe]]], SubQuery: [Query: [EditNodeAdd: [FormName: [Const: meta:source], Const: =, List: [VarValue: [Const: node], List: [Const: inet:email:header, List: [Const: to, Const: Visi Kensho <visi@vertex.link>]]]]]]]',
     'Query: [SetVarOper: [Const: x, DollarExpr: [ExprNode: [Const: 1, Const: /, Const: 3]]]]',
     'Query: [SetVarOper: [Const: x, DollarExpr: [ExprNode: [Const: 1, Const: *, Const: 3]]]]',
     'Query: [SetVarOper: [Const: x, DollarExpr: [ExprNode: [ExprNode: [Const: 1, Const: *, Const: 3], Const: +, Const: 2]]]]',
@@ -1338,18 +1340,18 @@ _ParseResults = [
     'Query: [LiftPropBy: [VarValue: [Const: form], Const: =, VarValue: [Const: valu]]]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: inet:dns*], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: inet:dns:*], isjoin=False]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:seen, Const: inet:dns:a]], isjoin=False]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:seen, Const: inet:dns*]], isjoin=False]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:source, Const: inet:dns:a]], isjoin=False]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:source, Const: inet:dns*]], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [VarValue: [Const: foo]], isjoin=False]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: inet:dns*], isjoin=True]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [Const: inet:dns:*], isjoin=True]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:seen, Const: inet:dns:a]], isjoin=True]',
-    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:seen, Const: inet:dns*]], isjoin=True]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:source, Const: inet:dns:a]], isjoin=True]',
+    'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [List: [Const: meta:source, Const: inet:dns*]], isjoin=True]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], FormPivot: [VarValue: [Const: foo]], isjoin=True]',
     'Query: [LiftPropBy: [Const: test:str, Const: =, Const: foobar], N1Walk: [Const: refs, Const: inet:dns:*], isjoin=False]',
-    'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: foo.com], PropPivot: [RelPropValue: [RelProp: [Const: zone]], List: [Const: meta:seen, Const: inet:dns:a]], isjoin=False]',
+    'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: foo.com], PropPivot: [RelPropValue: [RelProp: [Const: zone]], List: [Const: meta:source, Const: inet:dns:a]], isjoin=False]',
     'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: foo.com], PropPivot: [RelPropValue: [RelProp: [Const: zone]], VarValue: [Const: foo]], isjoin=False]',
-    'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: foo.com], PropPivot: [RelPropValue: [RelProp: [Const: zone]], List: [Const: meta:seen, Const: inet:dns:a]], isjoin=True]',
+    'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: foo.com], PropPivot: [RelPropValue: [RelProp: [Const: zone]], List: [Const: meta:source, Const: inet:dns:a]], isjoin=True]',
     'Query: [LiftPropBy: [Const: inet:fqdn, Const: =, Const: foo.com], PropPivot: [RelPropValue: [RelProp: [Const: zone]], VarValue: [Const: foo]], isjoin=True]',
     'Query: [LiftFormTag: [Const: test:*, TagName: [Const: foo]]]',
     'Query: [LiftFormTag: [Const: test:*, TagName: [Const: foo], Const: @=, Const: 2016]]',
@@ -1394,6 +1396,8 @@ _ParseResults = [
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: --=, List: [Const: foo, Const: bar]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ?++=, VarValue: [Const: baz]]]',
     'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditPropSetMulti: [RelProp: [Const: bar], Const: ?--=, SubQuery: [Query: [EditNodeAdd: [FormName: [Const: it:dev:str], Const: =, Const: foo]]]]]',
+    'Query: [SetVarOper: [Const: foo, List: [Const: notime]]]',
+    'Query: [SetVarOper: [Const: foo, List: [Const: nulltime]]]',
 ]
 
 class GrammarTest(s_t_utils.SynTest):

synapse/tests/test_lib_httpapi.py

@@ -1,5 +1,4 @@
 import ssl
-import json
 
 import aiohttp
 import aiohttp.client_exceptions as a_exc
@@ -8,6 +7,7 @@ import synapse.common as s_common
 import synapse.tools.backup as s_backup
 
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.link as s_link
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.version as s_version
@@ -671,7 +671,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        podes.append(json.loads(byts))
+                        podes.append(s_json.loads(byts))
 
                 self.eq(podes[0][0], ('inet:ipv4', 0x01020304))
 
@@ -685,7 +685,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        msgs.append(json.loads(byts))
+                        msgs.append(s_json.loads(byts))
                 podes = [m[1] for m in msgs if m[0] == 'node']
                 self.eq(podes[0][0], ('inet:ipv4', 0x05050505))
 
@@ -1322,7 +1322,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        mesg = json.loads(byts)
+                        mesg = s_json.loads(byts)
 
                         if mesg[0] == 'node':
                             node = mesg[1]
@@ -1337,7 +1337,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        mesg = json.loads(byts)
+                        mesg = s_json.loads(byts)
 
                         if mesg[0] == 'node':
                             node = mesg[1]
@@ -1354,7 +1354,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        node = json.loads(byts)
+                        node = s_json.loads(byts)
 
                     self.eq(0x01020304, node[0][1])
 
@@ -1365,7 +1365,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        node = json.loads(byts)
+                        node = s_json.loads(byts)
 
                     self.eq(0x01020304, node[0][1])
 
@@ -1385,8 +1385,8 @@ class HttpApiTest(s_tests.SynTest):
                                 break
 
                             try:
-                                node = json.loads(byts)
-                            except json.JSONDecodeError:
+                                node = s_json.loads(byts)
+                            except s_exc.BadJsonText:
                                 bufr = jstr
                                 break
 
@@ -1407,8 +1407,8 @@ class HttpApiTest(s_tests.SynTest):
                                 break
 
                             try:
-                                mesg = json.loads(byts)
-                            except json.JSONDecodeError:
+                                mesg = s_json.loads(byts)
+                            except s_exc.BadJsonText:
                                 bufr = jstr
                                 break
 
@@ -1427,7 +1427,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        mesg = json.loads(byts)
+                        mesg = s_json.loads(byts)
                         if mesg[0] == 'node':
                             task = core.boss.tasks.get(list(core.boss.tasks.keys())[0])
                             self.eq(core.view.iden, task.info.get('view'))
@@ -1445,7 +1445,7 @@ class HttpApiTest(s_tests.SynTest):
                         if not byts:
                             break
 
-                        mesg = json.loads(byts)
+                        mesg = s_json.loads(byts)
                         self.len(2, mesg)  # Is if roughly shaped like a node?
                         task = core.boss.tasks.get(list(core.boss.tasks.keys())[0])
                         break

synapse/tests/test_lib_json.py

@@ -0,0 +1,219 @@
+import io
+import json
+
+import orjson
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.json as s_json
+
+import synapse.tests.utils as s_test
+
+class JsonTest(s_test.SynTest):
+
+    async def test_lib_json_loads(self):
+        self.eq({'a': 'b'}, s_json.loads('{"a": "b"}'))
+
+        with self.raises(s_exc.BadJsonText) as exc:
+            s_json.loads('newp')
+        self.eq(exc.exception.get('mesg'), 'Expecting value: line 1 column 1 (char 0)')
+
+    async def test_lib_json_load(self):
+        with self.getTestDir() as dirn:
+
+            with s_common.genfile(dirn, 'file00') as file00:
+                file00.write(b'{"a": "b"}')
+
+            with s_common.genfile(dirn, 'file00') as file00:
+                self.eq({'a': 'b'}, s_json.load(file00))
+
+            with s_common.genfile(dirn, 'empty') as empty:
+                with self.raises(s_exc.BadJsonText) as exc:
+                    s_json.load(empty)
+                self.eq(exc.exception.get('mesg'), 'Expecting value: line 1 column 1 (char 0)')
+
+            buf = io.BytesIO(b'{"a": "b"}')
+            self.eq({'a': 'b'}, s_json.load(buf))
+
+            buf = io.StringIO('{"a": "b"}')
+            self.eq({'a': 'b'}, s_json.load(buf))
+
+    async def test_lib_json_load_surrogates(self):
+
+        inval = '{"a": "😀\ud83d\ude47"}'
+        outval = {'a': '😀\ud83d\ude47'}
+
+        # orjson.dumps fails because of the surrogate pairs
+        with self.raises(orjson.JSONDecodeError):
+            orjson.loads(inval)
+
+        # stdlib json.loads passes because of voodoo magic
+        self.eq(outval, json.loads(inval))
+
+        self.eq(outval, s_json.loads(inval))
+
+        buf = io.StringIO(inval)
+        self.eq(outval, s_json.load(buf))
+
+        buf = io.BytesIO(inval.encode('utf8', errors='surrogatepass'))
+        self.eq(outval, s_json.load(buf))
+
+    async def test_lib_json_dump_surrogates(self):
+        inval = {'a': '😀\ud83d\ude47'}
+        outval = b'{"a": "\\ud83d\\ude00\\ud83d\\ude47"}'
+
+        # orjson.dumps fails because of the surrogate pairs
+        with self.raises(TypeError):
+            orjson.dumps(inval)
+
+        # stdlib json.dumps passes because of voodoo magic
+        self.eq(outval.decode(), json.dumps(inval))
+
+        self.eq(outval, s_json.dumps(inval))
+        self.eq(outval + b'\n', s_json.dumps(inval, newline=True))
+
+        buf = io.BytesIO()
+        s_json.dump(inval, buf)
+        self.eq(outval, buf.getvalue())
+
+    async def test_lib_json_dumps(self):
+        self.eq(b'{"c":"d","a":"b"}', s_json.dumps({'c': 'd', 'a': 'b'}))
+        self.eq(b'{"a":"b","c":"d"}', s_json.dumps({'c': 'd', 'a': 'b'}, sort_keys=True))
+        self.eq(b'{\n  "c": "d",\n  "a": "b"\n}', s_json.dumps({'c': 'd', 'a': 'b'}, indent=True))
+        self.eq(b'{"c":"d","a":"b"}\n', s_json.dumps({'c': 'd', 'a': 'b'}, newline=True))
+
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.dumps({}.items())
+        self.eq(exc.exception.get('mesg'), 'Type is not JSON serializable: dict_items')
+
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.dumps({1: 'foo'})
+        self.eq(exc.exception.get('mesg'), 'Dict key must be str')
+
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.dumps({'\ud83d\ude47': {}.items()})
+        self.eq(exc.exception.get('mesg'), 'Object of type dict_items is not JSON serializable')
+
+        self.eq(b'"dict_items([])"', s_json.dumps({}.items(), default=str))
+
+    async def test_lib_json_dump(self):
+        with self.getTestDir() as dirn:
+            binfn = s_common.genpath(dirn, 'bin.json')
+
+            with open(binfn, 'wb') as binfp:
+                s_json.dump({'c': 'd', 'a': 'b'}, binfp)
+
+            with open(binfn, 'rb') as binfp:
+                self.eq(b'{"c":"d","a":"b"}', binfp.read())
+
+        buf = io.BytesIO()
+        s_json.dump({'c': 'd', 'a': 'b'}, buf)
+        self.eq(b'{"c":"d","a":"b"}', buf.getvalue())
+
+    async def test_jsload(self):
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'jsload.json') as fp:
+                fp.write(b'{"a":"b"}')
+
+            obj = s_json.jsload(dirn, 'jsload.json')
+            self.eq({'a': 'b'}, obj)
+
+            s_common.genfile(dirn, 'empty.json').close()
+            self.none(s_json.jsload(dirn, 'empty.json'))
+
+    async def test_jslines(self):
+        with self.getTestDir() as dirn:
+            with s_common.genfile(dirn, 'jslines.json') as fp:
+                fp.write(b'{"a":"b"}\n{"c":"d"}')
+
+            objs = [k for k in s_json.jslines(dirn, 'jslines.json')]
+            self.len(2, objs)
+            self.eq([{'a': 'b'}, {'c': 'd'}], objs)
+
+    async def test_jssave(self):
+        with self.getTestDir() as dirn:
+            s_json.jssave({'a': 'b'}, dirn, 'jssave.json')
+
+            with s_common.genfile(dirn, 'jssave.json') as fd:
+                data = fd.read()
+
+            self.eq(data, b'{\n  "a": "b"\n}')
+
+    async def test_lib_json_reqjsonsafe(self):
+        self.none(s_json.reqjsonsafe('foo'))
+        self.none(s_json.reqjsonsafe({'foo': 'bar'}))
+        self.none(s_json.reqjsonsafe(['foo', 'bar']))
+
+        buf = io.BytesIO()
+
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.reqjsonsafe(buf)
+        self.isin('Type is not JSON serializable: _io.BytesIO', exc.exception.get('mesg'))
+
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.reqjsonsafe({'foo': buf})
+        self.isin('Type is not JSON serializable: _io.BytesIO', exc.exception.get('mesg'))
+
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.reqjsonsafe(['foo', buf])
+        self.isin('Type is not JSON serializable: _io.BytesIO', exc.exception.get('mesg'))
+
+        items = (
+            (None, None),
+            (1234, None),
+            ('1234', None),
+            ({'asdf': 'haha'}, None),
+            ({'a': (1,), 'b': [{'': 4}, 56, None, {'t': True, 'f': False}, 'oh my']}, None),
+            (b'1234', s_exc.MustBeJsonSafe),
+            ({'a': 'a', 2: 2}, s_exc.MustBeJsonSafe),
+            ({'a', 'b', 'c'}, s_exc.MustBeJsonSafe),
+            (s_common.novalu, s_exc.MustBeJsonSafe),
+        )
+        for (item, eret) in items:
+            if eret is None:
+                self.none(s_json.reqjsonsafe(item))
+            else:
+                with self.raises(eret):
+                    s_json.reqjsonsafe(item)
+
+        text = '😀\ud83d\ude47'
+        s_json.reqjsonsafe(text)
+        with self.raises(s_exc.MustBeJsonSafe) as exc:
+            s_json.reqjsonsafe(text, strict=True)
+        self.eq(exc.exception.get('mesg'), 'str is not valid UTF-8: surrogates not allowed')
+
+    async def test_lib_json_data_at_rest(self):
+        async with self.getRegrCore('json-data') as core:
+            badjson = {
+                1: 'foo',
+                'foo': '😀\ud83d\ude47',
+            }
+
+            goodjson = {
+                '1': 'foo',
+                'foo': '😀',
+            }
+
+            # We can lift nodes with bad :data
+            nodes = await core.nodes('it:log:event')
+            self.len(1, nodes)
+            self.eq(nodes[0].get('data'), badjson)
+
+            iden = nodes[0].iden()
+
+            # We can't lift nodes with bad data by querying the prop directly
+            opts = {'vars': {'data': badjson}}
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('it:log:event:data=$data', opts=opts)
+
+            # We can't set nodes with bad data
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('[ it:log:event=* :data=$data ]', opts=opts)
+
+            # We can overwrite bad :data props
+            opts = {'vars': {'data': goodjson}}
+            nodes = await core.nodes('it:log:event:data [ :data=$data ]', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].iden(), iden)
+            self.eq(nodes[0].get('data'), goodjson)

synapse/tests/test_lib_multislabseqn.py

@@ -5,6 +5,7 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.coro as s_coro
+import synapse.lib.json as s_json
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.multislabseqn as s_multislabseqn
 
@@ -314,7 +315,7 @@ class MultiSlabSeqn(s_t_utils.SynTest):
             # Make a slab a non-dir
             slab0dirn = s_common.genpath(baddirn, f'seqn{"0" * 16}.lmdb')
             shutil.rmtree(slab0dirn)
-            s_common.jssave('{}', slab0dirn)
+            s_json.jssave('{}', slab0dirn)
 
             with self.getAsyncLoggerStream('synapse.lib.multislabseqn', 'non-directory') as stream:
                 async with await s_multislabseqn.MultiSlabSeqn.anit(baddirn) as msqn: