synapse 2.154.1__py311-none-any.whl → 2.156.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (74) hide show
  1. synapse/cmds/cortex.py +2 -14
  2. synapse/common.py +13 -36
  3. synapse/cortex.py +15 -508
  4. synapse/lib/ast.py +215 -22
  5. synapse/lib/cell.py +35 -8
  6. synapse/lib/certdir.py +11 -0
  7. synapse/lib/cmdr.py +0 -5
  8. synapse/lib/gis.py +2 -2
  9. synapse/lib/httpapi.py +14 -43
  10. synapse/lib/layer.py +64 -201
  11. synapse/lib/lmdbslab.py +11 -0
  12. synapse/lib/node.py +1 -3
  13. synapse/lib/parser.py +10 -0
  14. synapse/lib/slabseqn.py +2 -1
  15. synapse/lib/snap.py +121 -21
  16. synapse/lib/spooled.py +9 -0
  17. synapse/lib/storm.lark +23 -6
  18. synapse/lib/storm.py +16 -339
  19. synapse/lib/storm_format.py +5 -0
  20. synapse/lib/stormhttp.py +10 -1
  21. synapse/lib/stormlib/gen.py +1 -2
  22. synapse/lib/stormlib/gis.py +41 -0
  23. synapse/lib/stormlib/graph.py +2 -1
  24. synapse/lib/stormlib/stats.py +21 -2
  25. synapse/lib/stormlib/storm.py +16 -1
  26. synapse/lib/stormtypes.py +244 -16
  27. synapse/lib/types.py +16 -2
  28. synapse/lib/version.py +2 -2
  29. synapse/lib/view.py +118 -25
  30. synapse/models/base.py +2 -2
  31. synapse/models/inet.py +60 -30
  32. synapse/models/infotech.py +130 -8
  33. synapse/models/orgs.py +3 -0
  34. synapse/models/proj.py +3 -0
  35. synapse/models/risk.py +24 -6
  36. synapse/models/syn.py +0 -38
  37. synapse/tests/test_cmds_cortex.py +1 -1
  38. synapse/tests/test_cortex.py +70 -338
  39. synapse/tests/test_lib_agenda.py +19 -54
  40. synapse/tests/test_lib_aha.py +97 -0
  41. synapse/tests/test_lib_ast.py +596 -0
  42. synapse/tests/test_lib_grammar.py +30 -10
  43. synapse/tests/test_lib_httpapi.py +33 -49
  44. synapse/tests/test_lib_layer.py +19 -234
  45. synapse/tests/test_lib_lmdbslab.py +22 -0
  46. synapse/tests/test_lib_snap.py +9 -0
  47. synapse/tests/test_lib_spooled.py +4 -0
  48. synapse/tests/test_lib_storm.py +16 -309
  49. synapse/tests/test_lib_stormlib_gis.py +21 -0
  50. synapse/tests/test_lib_stormlib_stats.py +107 -20
  51. synapse/tests/test_lib_stormlib_storm.py +25 -0
  52. synapse/tests/test_lib_stormtypes.py +253 -8
  53. synapse/tests/test_lib_types.py +40 -0
  54. synapse/tests/test_lib_view.py +6 -13
  55. synapse/tests/test_model_base.py +1 -1
  56. synapse/tests/test_model_inet.py +15 -0
  57. synapse/tests/test_model_infotech.py +110 -0
  58. synapse/tests/test_model_orgs.py +10 -0
  59. synapse/tests/test_model_person.py +0 -3
  60. synapse/tests/test_model_proj.py +2 -1
  61. synapse/tests/test_model_risk.py +24 -0
  62. synapse/tests/test_model_syn.py +20 -34
  63. synapse/tests/test_tools_csvtool.py +2 -1
  64. synapse/tests/test_tools_feed.py +4 -30
  65. synapse/tools/csvtool.py +2 -1
  66. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/METADATA +9 -9
  67. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/RECORD +70 -72
  68. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/WHEEL +1 -1
  69. synapse/cmds/cron.py +0 -726
  70. synapse/cmds/trigger.py +0 -319
  71. synapse/tests/test_cmds_cron.py +0 -453
  72. synapse/tests/test_cmds_trigger.py +0 -176
  73. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/LICENSE +0 -0
  74. {synapse-2.154.1.dist-info → synapse-2.156.0.dist-info}/top_level.txt +0 -0
synapse/lib/stormtypes.py CHANGED
@@ -6317,6 +6317,29 @@ class Layer(Prim):
6317
6317
  {'name': 'propname', 'type': 'str', 'desc': 'The property or form name to look up.', },
6318
6318
  {'name': 'maxsize', 'type': 'int', 'desc': 'The maximum number of rows to look up.',
6319
6319
  'default': None, },
6320
+ {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
6321
+ 'desc': 'A specific value of the property to look up.', },
6322
+ ),
6323
+ 'returns': {'type': 'int', 'desc': 'The count of rows.', }}},
6324
+ {'name': 'getPropArrayCount',
6325
+ 'desc': 'Get the number of individual value rows in the layer for the given array property name.',
6326
+ 'type': {'type': 'function', '_funcname': '_methGetPropArrayCount',
6327
+ 'args': (
6328
+ {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
6329
+ {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
6330
+ 'desc': 'A specific value in the array property to look up.', },
6331
+ ),
6332
+ 'returns': {'type': 'int', 'desc': 'The count of rows.', }}},
6333
+ {'name': 'getTagPropCount',
6334
+ 'desc': 'Get the number of rows in the layer for the given tag property.',
6335
+ 'type': {'type': 'function', '_funcname': '_methGetTagPropCount',
6336
+ 'args': (
6337
+ {'name': 'tag', 'type': 'str', 'desc': 'The tag to look up.', },
6338
+ {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
6339
+ {'name': 'form', 'type': 'str', 'default': None,
6340
+ 'desc': 'The optional form to look up.', },
6341
+ {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
6342
+ 'desc': 'A specific value of the property to look up.', },
6320
6343
  ),
6321
6344
  'returns': {'type': 'int', 'desc': 'The count of rows.', }}},
6322
6345
  {'name': 'getFormCounts', 'desc': '''
@@ -6514,6 +6537,8 @@ class Layer(Prim):
6514
6537
  'liftByProp': self.liftByProp,
6515
6538
  'getTagCount': self._methGetTagCount,
6516
6539
  'getPropCount': self._methGetPropCount,
6540
+ 'getTagPropCount': self._methGetTagPropCount,
6541
+ 'getPropArrayCount': self._methGetPropArrayCount,
6517
6542
  'getFormCounts': self._methGetFormcount,
6518
6543
  'getStorNode': self.getStorNode,
6519
6544
  'getStorNodes': self.getStorNodes,
@@ -6678,7 +6703,7 @@ class Layer(Prim):
6678
6703
  return await layr.getTagCount(tagname, formname=formname)
6679
6704
 
6680
6705
  @stormfunc(readonly=True)
6681
- async def _methGetPropCount(self, propname, maxsize=None):
6706
+ async def _methGetPropCount(self, propname, maxsize=None, valu=undef):
6682
6707
  propname = await tostr(propname)
6683
6708
  maxsize = await toint(maxsize, noneok=True)
6684
6709
 
@@ -6691,13 +6716,86 @@ class Layer(Prim):
6691
6716
  await self.runt.reqUserCanReadLayer(layriden)
6692
6717
  layr = self.runt.snap.core.getLayer(layriden)
6693
6718
 
6719
+ if valu is undef:
6720
+ if prop.isform:
6721
+ return await layr.getPropCount(prop.name, None, maxsize=maxsize)
6722
+
6723
+ if prop.isuniv:
6724
+ return await layr.getPropCount(None, prop.name, maxsize=maxsize)
6725
+
6726
+ return await layr.getPropCount(prop.form.name, prop.name, maxsize=maxsize)
6727
+
6728
+ valu = await toprim(valu)
6729
+ norm, info = prop.type.norm(valu)
6730
+
6731
+ if prop.isform:
6732
+ return layr.getPropValuCount(prop.name, None, prop.type.stortype, norm)
6733
+
6734
+ if prop.isuniv:
6735
+ return layr.getPropValuCount(None, prop.name, prop.type.stortype, norm)
6736
+
6737
+ return layr.getPropValuCount(prop.form.name, prop.name, prop.type.stortype, norm)
6738
+
6739
+ @stormfunc(readonly=True)
6740
+ async def _methGetPropArrayCount(self, propname, valu=undef):
6741
+ propname = await tostr(propname)
6742
+
6743
+ prop = self.runt.snap.core.model.prop(propname)
6744
+ if prop is None:
6745
+ mesg = f'No property named {propname}'
6746
+ raise s_exc.NoSuchProp(mesg=mesg)
6747
+
6748
+ if not prop.type.isarray:
6749
+ mesg = f'Property is not an array type: {prop.type.name}.'
6750
+ raise s_exc.BadTypeValu(mesg=mesg)
6751
+
6752
+ layriden = self.valu.get('iden')
6753
+ await self.runt.reqUserCanReadLayer(layriden)
6754
+ layr = self.runt.snap.core.getLayer(layriden)
6755
+
6756
+ if valu is undef:
6757
+ if prop.isform:
6758
+ return await layr.getPropArrayCount(prop.name, None)
6759
+
6760
+ if prop.isuniv:
6761
+ return await layr.getPropArrayCount(None, prop.name)
6762
+
6763
+ return await layr.getPropArrayCount(prop.form.name, prop.name)
6764
+
6765
+ valu = await toprim(valu)
6766
+ atyp = prop.type.arraytype
6767
+ norm, info = atyp.norm(valu)
6768
+
6694
6769
  if prop.isform:
6695
- return await layr.getPropCount(prop.name, None, maxsize=maxsize)
6770
+ return layr.getPropArrayValuCount(prop.name, None, atyp.stortype, norm)
6696
6771
 
6697
6772
  if prop.isuniv:
6698
- return await layr.getUnivPropCount(prop.name, maxsize=maxsize)
6773
+ return layr.getPropArrayValuCount(None, prop.name, atyp.stortype, norm)
6774
+
6775
+ return layr.getPropArrayValuCount(prop.form.name, prop.name, atyp.stortype, norm)
6776
+
6777
+ @stormfunc(readonly=True)
6778
+ async def _methGetTagPropCount(self, tag, propname, form=None, valu=undef):
6779
+ tag = await tostr(tag)
6780
+ propname = await tostr(propname)
6781
+ form = await tostr(form, noneok=True)
6782
+
6783
+ prop = self.runt.snap.core.model.getTagProp(propname)
6784
+ if prop is None:
6785
+ mesg = f'No tag property named {propname}'
6786
+ raise s_exc.NoSuchTagProp(name=propname, mesg=mesg)
6787
+
6788
+ layriden = self.valu.get('iden')
6789
+ await self.runt.reqUserCanReadLayer(layriden)
6790
+ layr = self.runt.snap.core.getLayer(layriden)
6791
+
6792
+ if valu is undef:
6793
+ return await layr.getTagPropCount(form, tag, prop.name)
6794
+
6795
+ valu = await toprim(valu)
6796
+ norm, info = prop.type.norm(valu)
6699
6797
 
6700
- return await layr.getPropCount(prop.form.name, prop.name, maxsize=maxsize)
6798
+ return layr.getTagPropValuCount(form, tag, prop.name, prop.type.stortype, norm)
6701
6799
 
6702
6800
  @stormfunc(readonly=True)
6703
6801
  async def _methLayerEdits(self, offs=0, wait=True, size=None):
@@ -7021,6 +7119,68 @@ class View(Prim):
7021
7119
  'returns':
7022
7120
  {'type': 'dict',
7023
7121
  'desc': "Dictionary containing form names and the count of the nodes in the View's Layers.", }}},
7122
+
7123
+ {'name': 'getPropCount',
7124
+ 'desc': '''
7125
+ Get the number of nodes in the View with a specific property and optional value.
7126
+
7127
+ Notes:
7128
+ This is a fast approximate count calculated by summing the number of
7129
+ nodes with the property value in each layer of the view. Property values
7130
+ which are overwritten by different values in higher layers will still
7131
+ be included in the count.
7132
+ ''',
7133
+ 'type': {'type': 'function', '_funcname': '_methGetPropCount',
7134
+ 'args': (
7135
+ {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
7136
+ {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
7137
+ 'desc': 'The value of the property to look up.', },
7138
+ ),
7139
+ 'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
7140
+
7141
+ {'name': 'getPropArrayCount',
7142
+ 'desc': '''
7143
+ Get the number of invidivual array property values in the View for the given array property name.
7144
+
7145
+ Notes:
7146
+ This is a fast approximate count calculated by summing the number of
7147
+ array property values in each layer of the view. Property values
7148
+ which are overwritten by different values in higher layers will
7149
+ still be included in the count.
7150
+ ''',
7151
+ 'type': {'type': 'function', '_funcname': '_methGetPropArrayCount',
7152
+ 'args': (
7153
+ {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
7154
+ {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
7155
+ 'desc': 'The value in the array property to look up.', },
7156
+ ),
7157
+ 'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
7158
+
7159
+ {'name': 'getTagPropCount',
7160
+ 'desc': '''
7161
+ Get the number of nodes in the View with the given tag property and optional value.
7162
+
7163
+ Notes:
7164
+ This is a fast approximate count calculated by summing the number of
7165
+ nodes with the tag property value in each layer of the view.
7166
+ Values which are overwritten by different values in higher layers
7167
+ will still be included in the count.
7168
+ ''',
7169
+ 'type': {'type': 'function', '_funcname': '_methGetTagPropCount',
7170
+ 'args': (
7171
+ {'name': 'tag', 'type': 'str', 'desc': 'The tag to look up.', },
7172
+ {'name': 'propname', 'type': 'str', 'desc': 'The property name to look up.', },
7173
+ {'name': 'form', 'type': 'str', 'default': None,
7174
+ 'desc': 'The optional form to look up.', },
7175
+ {'name': 'valu', 'type': 'any', 'default': '$lib.undef',
7176
+ 'desc': 'The value of the property to look up.', },
7177
+ ),
7178
+ 'returns': {'type': 'int', 'desc': 'The count of nodes.', }}},
7179
+
7180
+ {'name': 'detach', 'desc': 'Detach the view from its parent. WARNING: This cannot be reversed.',
7181
+ 'type': {'type': 'function', '_funcname': 'detach',
7182
+ 'args': (),
7183
+ 'returns': {'type': 'null', }}},
7024
7184
  )
7025
7185
  _storm_typename = 'view'
7026
7186
  _ismutable = False
@@ -7047,12 +7207,16 @@ class View(Prim):
7047
7207
  'pack': self._methViewPack,
7048
7208
  'repr': self._methViewRepr,
7049
7209
  'merge': self._methViewMerge,
7210
+ 'detach': self.detach,
7050
7211
  'addNode': self.addNode,
7051
7212
  'getEdges': self._methGetEdges,
7052
7213
  'wipeLayer': self._methWipeLayer,
7053
7214
  'addNodeEdits': self._methAddNodeEdits,
7054
7215
  'getEdgeVerbs': self._methGetEdgeVerbs,
7055
7216
  'getFormCounts': self._methGetFormcount,
7217
+ 'getPropCount': self._methGetPropCount,
7218
+ 'getTagPropCount': self._methGetTagPropCount,
7219
+ 'getPropArrayCount': self._methGetPropArrayCount,
7056
7220
  }
7057
7221
 
7058
7222
  async def addNode(self, form, valu, props=None):
@@ -7080,6 +7244,15 @@ class View(Prim):
7080
7244
  else:
7081
7245
  await view.addNode(form, valu, props=props, user=self.runt.user)
7082
7246
 
7247
+ async def detach(self):
7248
+
7249
+ view = self._reqView()
7250
+ if not self.runt.isAdmin(gateiden=view.iden):
7251
+ mesg = 'You must be an admin of the view to detach.'
7252
+ raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
7253
+
7254
+ await view.detach()
7255
+
7083
7256
  async def _methAddNodeEdits(self, edits):
7084
7257
  useriden = self.runt.user.iden
7085
7258
  viewiden = self.valu.get('iden')
@@ -7097,6 +7270,53 @@ class View(Prim):
7097
7270
  todo = s_common.todo('getFormCounts')
7098
7271
  return await self.viewDynCall(todo, ('view', 'read'))
7099
7272
 
7273
+ @stormfunc(readonly=True)
7274
+ async def _methGetPropCount(self, propname, valu=undef):
7275
+ propname = await tostr(propname)
7276
+
7277
+ if valu is undef:
7278
+ valu = s_common.novalu
7279
+ else:
7280
+ valu = await toprim(valu)
7281
+
7282
+ viewiden = self.valu.get('iden')
7283
+ self.runt.confirm(('view', 'read'), gateiden=viewiden)
7284
+ view = self.runt.snap.core.getView(viewiden)
7285
+
7286
+ return await view.getPropCount(propname, valu=valu)
7287
+
7288
+ @stormfunc(readonly=True)
7289
+ async def _methGetTagPropCount(self, tag, propname, form=None, valu=undef):
7290
+ tag = await tostr(tag)
7291
+ propname = await tostr(propname)
7292
+ form = await tostr(form, noneok=True)
7293
+
7294
+ if valu is undef:
7295
+ valu = s_common.novalu
7296
+ else:
7297
+ valu = await toprim(valu)
7298
+
7299
+ viewiden = self.valu.get('iden')
7300
+ self.runt.confirm(('view', 'read'), gateiden=viewiden)
7301
+ view = self.runt.snap.core.getView(viewiden)
7302
+
7303
+ return await view.getTagPropCount(form, tag, propname, valu=valu)
7304
+
7305
+ @stormfunc(readonly=True)
7306
+ async def _methGetPropArrayCount(self, propname, valu=undef):
7307
+ propname = await tostr(propname)
7308
+
7309
+ if valu is undef:
7310
+ valu = s_common.novalu
7311
+ else:
7312
+ valu = await toprim(valu)
7313
+
7314
+ viewiden = self.valu.get('iden')
7315
+ self.runt.confirm(('view', 'read'), gateiden=viewiden)
7316
+ view = self.runt.snap.core.getView(viewiden)
7317
+
7318
+ return await view.getPropArrayCount(propname, valu=valu)
7319
+
7100
7320
  @stormfunc(readonly=True)
7101
7321
  async def _methGetEdges(self, verb=None):
7102
7322
  verb = await toprim(verb)
@@ -7127,6 +7347,9 @@ class View(Prim):
7127
7347
  async def _methViewGet(self, name, defv=None):
7128
7348
  return self.valu.get(name, defv)
7129
7349
 
7350
+ def _reqView(self):
7351
+ return self.runt.snap.core.reqView(self.valu.get('iden'))
7352
+
7130
7353
  async def _methViewSet(self, name, valu):
7131
7354
 
7132
7355
  name = await tostr(name)
@@ -7137,14 +7360,17 @@ class View(Prim):
7137
7360
  else:
7138
7361
  valu = await tostr(await toprim(valu), noneok=True)
7139
7362
 
7363
+ if name == 'parent' and valu is not None:
7364
+ self.runt.snap.core.reqView(valu, mesg='The parent view must already exist.')
7365
+ self.runt.confirm(('view', 'read'), gateiden=valu)
7366
+ self.runt.confirm(('view', 'fork'), gateiden=valu)
7367
+
7140
7368
  elif name == 'nomerge':
7141
7369
  valu = await tobool(valu)
7370
+
7142
7371
  elif name == 'layers':
7143
7372
 
7144
- view = self.runt.snap.core.getView(self.valu.get('iden'))
7145
- if view is None: # pragma: no cover
7146
- mesg = f'No view with iden: {self.valu.get("iden")}'
7147
- raise s_exc.NoSuchView(mesg=mesg, iden=self.valu.get('iden'))
7373
+ view = self._reqView()
7148
7374
 
7149
7375
  layers = await toprim(valu)
7150
7376
  layers = tuple(str(x) for x in layers)
@@ -7170,8 +7396,11 @@ class View(Prim):
7170
7396
  mesg = f'View does not support setting: {name}'
7171
7397
  raise s_exc.BadOptValu(mesg=mesg)
7172
7398
 
7173
- todo = s_common.todo('setViewInfo', name, valu)
7174
- valu = await self.viewDynCall(todo, ('view', 'set', name))
7399
+ view = self.runt.snap.core.reqView(self.valu.get('iden'))
7400
+
7401
+ self.runt.confirm(('view', 'set', name), gateiden=view.iden)
7402
+ await view.setViewInfo(name, valu)
7403
+
7175
7404
  self.valu[name] = valu
7176
7405
 
7177
7406
  @stormfunc(readonly=True)
@@ -7202,10 +7431,9 @@ class View(Prim):
7202
7431
  useriden = self.runt.user.iden
7203
7432
  viewiden = self.valu.get('iden')
7204
7433
 
7205
- gatekeys = (
7206
- (useriden, ('view', 'add'), None),
7207
- (useriden, ('view', 'read'), viewiden),
7208
- )
7434
+ self.runt.confirm(('view', 'add'))
7435
+ self.runt.confirm(('view', 'read'), gateiden=viewiden)
7436
+ self.runt.confirm(('view', 'fork'), gateiden=viewiden)
7209
7437
 
7210
7438
  ldef = {'creator': self.runt.user.iden}
7211
7439
  vdef = {'creator': self.runt.user.iden}
@@ -7213,9 +7441,9 @@ class View(Prim):
7213
7441
  if name is not None:
7214
7442
  vdef['name'] = name
7215
7443
 
7216
- todo = s_common.todo('fork', ldef=ldef, vdef=vdef)
7444
+ view = self.runt.snap.core.reqView(viewiden)
7217
7445
 
7218
- newv = await self.runt.dyncall(viewiden, todo, gatekeys=gatekeys)
7446
+ newv = await view.fork(ldef=ldef, vdef=vdef)
7219
7447
 
7220
7448
  return View(self.runt, newv, path=self.path)
7221
7449
 
synapse/lib/types.py CHANGED
@@ -929,6 +929,7 @@ class Int(IntBase):
929
929
  _opt_defs = (
930
930
  ('size', 8), # type: ignore # Set the storage size of the integer type in bytes.
931
931
  ('signed', True),
932
+ ('enums:strict', True),
932
933
 
933
934
  # Note: currently unused
934
935
  ('fmt', '%d'), # Set to an integer compatible format string to control repr.
@@ -952,6 +953,8 @@ class Int(IntBase):
952
953
  self.enumnorm = {}
953
954
  self.enumrepr = {}
954
955
 
956
+ self.enumstrict = self.opts.get('enums:strict')
957
+
955
958
  enums = self.opts.get('enums')
956
959
  if enums is not None:
957
960
  self.enumrepr.update(dict(enums))
@@ -1031,7 +1034,7 @@ class Int(IntBase):
1031
1034
  mesg = f'value is above max={self.maxval}'
1032
1035
  raise s_exc.BadTypeValu(valu=repr(valu), name=self.name, mesg=mesg)
1033
1036
 
1034
- if self.enumrepr and valu not in self.enumrepr:
1037
+ if self.enumrepr and self.enumstrict and valu not in self.enumrepr:
1035
1038
  mesg = 'Value is not a valid enum value.'
1036
1039
  raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg)
1037
1040
 
@@ -1761,8 +1764,19 @@ class Taxonomy(Str):
1761
1764
  self.setNormFunc(tuple, self._normPyList)
1762
1765
  self.taxon = self.modl.type('taxon')
1763
1766
 
1767
+ def _ctorCmprPref(self, valu):
1768
+ norm = self._normForLift(valu)
1769
+
1770
+ def cmpr(valu):
1771
+ return valu.startswith(norm)
1772
+
1773
+ return cmpr
1774
+
1764
1775
  def _normForLift(self, valu):
1765
- return self.norm(valu)[0]
1776
+ norm = self.norm(valu)[0]
1777
+ if isinstance(valu, str) and not valu.strip().endswith('.'):
1778
+ return norm.rstrip('.')
1779
+ return norm
1766
1780
 
1767
1781
  def _normPyList(self, valu):
1768
1782
 
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 154, 1)
226
+ version = (2, 156, 0)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = '596a72e1ce5102e18f289394f292c81bc65a3ff7'
228
+ commit = '918a46d769574c99d1510654c1c2aaa13c3fda2c'
synapse/lib/view.py CHANGED
@@ -144,6 +144,22 @@ class View(s_nexus.Pusher): # type: ignore
144
144
  self.trigtask = None
145
145
  await self.initTrigTask()
146
146
 
147
+ @s_nexus.Pusher.onPushAuto('view:detach')
148
+ async def detach(self):
149
+ '''
150
+ Detach the view from its parent but do not change the layers.
151
+ ( this is not reversible! )
152
+ '''
153
+ if not self.parent:
154
+ mesg = 'A view with no parent is already detached.'
155
+ raise s_exc.BadArg(mesg=mesg)
156
+
157
+ self.parent = None
158
+ await self.info.pop('parent')
159
+
160
+ await self.core.feedBeholder('view:set', {'iden': self.iden, 'name': 'parent', 'valu': None},
161
+ gates=[self.iden, self.layers[0].iden])
162
+
147
163
  async def mergeStormIface(self, name, todo):
148
164
  '''
149
165
  Allow an interface which specifies a generator use case to yield
@@ -382,6 +398,95 @@ class View(s_nexus.Pusher): # type: ignore
382
398
  counts[name] += valu
383
399
  return counts
384
400
 
401
+ async def getPropCount(self, propname, valu=s_common.novalu):
402
+ prop = self.core.model.prop(propname)
403
+ if prop is None:
404
+ mesg = f'No property named {propname}'
405
+ raise s_exc.NoSuchProp(mesg=mesg)
406
+
407
+ count = 0
408
+ formname = None
409
+ propname = None
410
+
411
+ if prop.isform:
412
+ formname = prop.name
413
+ else:
414
+ propname = prop.name
415
+ if not prop.isuniv:
416
+ formname = prop.form.name
417
+
418
+ if valu is s_common.novalu:
419
+ for layr in self.layers:
420
+ await asyncio.sleep(0)
421
+ count += await layr.getPropCount(formname, propname)
422
+ return count
423
+
424
+ norm, info = prop.type.norm(valu)
425
+
426
+ for layr in self.layers:
427
+ await asyncio.sleep(0)
428
+ count += layr.getPropValuCount(formname, propname, prop.type.stortype, norm)
429
+
430
+ return count
431
+
432
+ async def getTagPropCount(self, form, tag, propname, valu=s_common.novalu):
433
+ prop = self.core.model.getTagProp(propname)
434
+ if prop is None:
435
+ mesg = f'No tag property named {propname}'
436
+ raise s_exc.NoSuchTagProp(name=propname, mesg=mesg)
437
+
438
+ count = 0
439
+
440
+ if valu is s_common.novalu:
441
+ for layr in self.layers:
442
+ await asyncio.sleep(0)
443
+ count += await layr.getTagPropCount(form, tag, prop.name)
444
+ return count
445
+
446
+ norm, info = prop.type.norm(valu)
447
+
448
+ for layr in self.layers:
449
+ await asyncio.sleep(0)
450
+ count += layr.getTagPropValuCount(form, tag, prop.name, prop.type.stortype, norm)
451
+
452
+ return count
453
+
454
+ async def getPropArrayCount(self, propname, valu=s_common.novalu):
455
+ prop = self.core.model.prop(propname)
456
+ if prop is None:
457
+ mesg = f'No property named {propname}'
458
+ raise s_exc.NoSuchProp(mesg=mesg)
459
+
460
+ if not prop.type.isarray:
461
+ mesg = f'Property is not an array type: {prop.type.name}.'
462
+ raise s_exc.BadTypeValu(mesg=mesg)
463
+
464
+ count = 0
465
+ formname = None
466
+ propname = None
467
+
468
+ if prop.isform:
469
+ formname = prop.name
470
+ else:
471
+ propname = prop.name
472
+ if not prop.isuniv:
473
+ formname = prop.form.name
474
+
475
+ if valu is s_common.novalu:
476
+ for layr in self.layers:
477
+ await asyncio.sleep(0)
478
+ count += await layr.getPropArrayCount(formname, propname)
479
+ return count
480
+
481
+ atyp = prop.type.arraytype
482
+ norm, info = atyp.norm(valu)
483
+
484
+ for layr in self.layers:
485
+ await asyncio.sleep(0)
486
+ count += layr.getPropArrayValuCount(formname, propname, atyp.stortype, norm)
487
+
488
+ return count
489
+
385
490
  async def getEdgeVerbs(self):
386
491
 
387
492
  async with await s_spooled.Set.anit(dirn=self.core.dirn, cell=self.core) as vset:
@@ -515,22 +620,20 @@ class View(s_nexus.Pusher): # type: ignore
515
620
 
516
621
  mode = opts.get('mode', 'storm')
517
622
  editformat = opts.get('editformat', 'nodeedits')
518
- if editformat not in ('nodeedits', 'splices', 'count', 'none'):
519
- raise s_exc.BadConfValu(mesg='editformat')
520
-
521
- if editformat == 'splices':
522
- s_common.deprdate('storm option editformat=splices', s_common._splicedepr)
623
+ if editformat not in ('nodeedits', 'count', 'none'):
624
+ raise s_exc.BadConfValu(mesg=f'invalid edit format, got {editformat}', name='editformat', valu=editformat)
523
625
 
524
626
  texthash = hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
525
627
 
526
628
  async def runStorm():
527
629
  cancelled = False
528
630
  tick = s_common.now()
631
+ abstick = s_common.mononow()
529
632
  count = 0
530
633
  try:
531
634
 
532
635
  # Always start with an init message.
533
- await chan.put(('init', {'tick': tick, 'text': text,
636
+ await chan.put(('init', {'tick': tick, 'text': text, 'abstick': abstick,
534
637
  'hash': texthash, 'task': synt.iden}))
535
638
 
536
639
  # Try text parsing. If this fails, we won't be able to get a storm
@@ -582,9 +685,10 @@ class View(s_nexus.Pusher): # type: ignore
582
685
 
583
686
  finally:
584
687
  if not cancelled:
585
- tock = s_common.now()
586
- took = tock - tick
587
- await chan.put(('fini', {'tock': tock, 'took': took, 'count': count}))
688
+ abstock = s_common.mononow()
689
+ abstook = abstock - abstick
690
+ tock = tick + abstook
691
+ await chan.put(('fini', {'tock': tock, 'abstock': abstock, 'took': abstook, 'count': count, }))
588
692
 
589
693
  await synt.worker(runStorm(), name='runstorm')
590
694
 
@@ -611,18 +715,11 @@ class View(s_nexus.Pusher): # type: ignore
611
715
  if editformat == 'none':
612
716
  continue
613
717
 
614
- if editformat == 'count':
615
- count = sum(len(edit[2]) for edit in mesg[1].get('edits', ()))
616
- mesg = ('node:edits:count', {'count': count})
617
- yield mesg
618
- continue
619
-
620
- assert editformat == 'splices'
718
+ assert editformat == 'count'
621
719
 
622
- nodeedits = mesg[1].get('edits', [()])
623
- async for _, splice in self.layers[0].makeSplices(0, nodeedits, None):
624
- if not show or splice[0] in show:
625
- yield splice
720
+ count = sum(len(edit[2]) for edit in mesg[1].get('edits', ()))
721
+ mesg = ('node:edits:count', {'count': count})
722
+ yield mesg
626
723
  continue
627
724
 
628
725
  if kind == 'fini':
@@ -672,11 +769,7 @@ class View(s_nexus.Pusher): # type: ignore
672
769
 
673
770
  if name == 'parent':
674
771
 
675
- parent = self.core.getView(valu)
676
- if parent is None:
677
- mesg = 'The parent view must already exist.'
678
- raise s_exc.NoSuchView(mesg=mesg)
679
-
772
+ parent = self.core.reqView(valu, mesg='The parent view must already exist.')
680
773
  if parent.iden == self.iden:
681
774
  mesg = 'A view may not have parent set to itself.'
682
775
  raise s_exc.BadArg(mesg=mesg)
synapse/models/base.py CHANGED
@@ -87,10 +87,10 @@ class BaseModule(s_module.CoreModule):
87
87
  ('graph:timeedge', ('timeedge', {}), {
88
88
  'doc': 'A generic digraph time edge to show relationships outside the model.'}),
89
89
 
90
- ('meta:priority', ('int', {'enums': prioenums}), {
90
+ ('meta:priority', ('int', {'enums': prioenums, 'enums:strict': False}), {
91
91
  'doc': 'A generic priority enumeration.'}),
92
92
 
93
- ('meta:severity', ('int', {'enums': prioenums}), {
93
+ ('meta:severity', ('int', {'enums': prioenums, 'enums:strict': False}), {
94
94
  'doc': 'A generic severity enumeration.'}),
95
95
 
96
96
  ('meta:sophistication', ('int', {'enums': sophenums}), {