synapse 2.197.0__py311-none-any.whl → 2.198.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (44) hide show
  1. synapse/axon.py +3 -0
  2. synapse/common.py +3 -0
  3. synapse/cortex.py +1 -3
  4. synapse/lib/aha.py +3 -0
  5. synapse/lib/ast.py +277 -165
  6. synapse/lib/auth.py +39 -11
  7. synapse/lib/cell.py +22 -4
  8. synapse/lib/hive.py +2 -1
  9. synapse/lib/hiveauth.py +10 -1
  10. synapse/lib/jsonstor.py +6 -5
  11. synapse/lib/layer.py +6 -5
  12. synapse/lib/node.py +10 -4
  13. synapse/lib/parser.py +46 -21
  14. synapse/lib/schemas.py +13 -0
  15. synapse/lib/snap.py +68 -26
  16. synapse/lib/storm.lark +13 -11
  17. synapse/lib/storm.py +1 -1
  18. synapse/lib/storm_format.py +3 -2
  19. synapse/lib/stormtypes.py +13 -4
  20. synapse/lib/version.py +2 -2
  21. synapse/models/infotech.py +18 -0
  22. synapse/models/risk.py +9 -0
  23. synapse/models/syn.py +18 -2
  24. synapse/tests/files/stormpkg/badendpoints.yaml +7 -0
  25. synapse/tests/files/stormpkg/testpkg.yaml +8 -0
  26. synapse/tests/test_cortex.py +108 -0
  27. synapse/tests/test_lib_aha.py +12 -2
  28. synapse/tests/test_lib_ast.py +57 -0
  29. synapse/tests/test_lib_auth.py +143 -2
  30. synapse/tests/test_lib_grammar.py +54 -2
  31. synapse/tests/test_lib_lmdbslab.py +24 -0
  32. synapse/tests/test_lib_storm.py +20 -0
  33. synapse/tests/test_lib_stormlib_macro.py +3 -3
  34. synapse/tests/test_lib_stormtypes.py +14 -2
  35. synapse/tests/test_model_infotech.py +13 -0
  36. synapse/tests/test_model_risk.py +6 -0
  37. synapse/tests/test_model_syn.py +58 -0
  38. synapse/tests/test_tools_genpkg.py +10 -0
  39. synapse/tools/hive/load.py +1 -0
  40. {synapse-2.197.0.dist-info → synapse-2.198.0.dist-info}/METADATA +1 -1
  41. {synapse-2.197.0.dist-info → synapse-2.198.0.dist-info}/RECORD +44 -43
  42. {synapse-2.197.0.dist-info → synapse-2.198.0.dist-info}/LICENSE +0 -0
  43. {synapse-2.197.0.dist-info → synapse-2.198.0.dist-info}/WHEEL +0 -0
  44. {synapse-2.197.0.dist-info → synapse-2.198.0.dist-info}/top_level.txt +0 -0
synapse/lib/auth.py CHANGED
@@ -375,11 +375,32 @@ class Auth(s_nexus.Pusher):
375
375
 
376
376
  await self.fire('cell:beholder', **behold)
377
377
 
378
- @s_nexus.Pusher.onPushAuto('user:info')
379
378
  async def setUserInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
380
379
 
381
380
  user = await self.reqUser(iden)
382
381
 
382
+ if name == 'locked' and not valu and user.isArchived():
383
+ raise s_exc.BadArg(mesg='Cannot unlock archived user.', user=iden, username=user.name)
384
+
385
+ if name in ('locked', 'archived') and not valu:
386
+ self.checkUserLimit()
387
+
388
+ await self._push('user:info', iden, name, valu, gateiden=gateiden, logged=logged, mesg=mesg)
389
+
390
+ @s_nexus.Pusher.onPush('user:info')
391
+ async def _setUserInfo(self, iden, name, valu, gateiden=None, logged=True, mesg=None):
392
+ user = await self.reqUser(iden)
393
+
394
+ if self.nexsroot and self.nexsroot.cell.nexsvers >= (2, 198):
395
+ # If the nexus version is less than 2.197 then the leader hasn't been upgraded yet and
396
+ # we don't want to get into a schism because we're bouncing edits and the leader is
397
+ # applying them.
398
+ if name == 'locked' and not valu and user.isArchived():
399
+ return
400
+
401
+ if name in ('locked', 'archived') and not valu:
402
+ self.checkUserLimit()
403
+
383
404
  if gateiden is not None:
384
405
  info = user.genGateInfo(gateiden)
385
406
  info[name] = s_msgpack.deepcopy(valu)
@@ -392,9 +413,6 @@ class Auth(s_nexus.Pusher):
392
413
  user.info[name] = s_msgpack.deepcopy(valu)
393
414
  self.userdefs.set(iden, user.info)
394
415
 
395
- if name in ('locked', 'archived') and not valu:
396
- self.checkUserLimit()
397
-
398
416
  if mesg is None:
399
417
  mesg = {
400
418
  'iden': iden,
@@ -720,6 +738,16 @@ class Auth(s_nexus.Pusher):
720
738
  self.roleidenbyname.delete(role.name)
721
739
  await self.feedBeholder('role:del', {'iden': iden})
722
740
 
741
+ def clearAuthCache(self):
742
+ '''
743
+ Clear all auth caches.
744
+ '''
745
+ self.userbyidencache.clear()
746
+ self.useridenbynamecache.clear()
747
+ self.rolebyidencache.clear()
748
+ self.roleidenbynamecache.clear()
749
+ self.authgates.clear()
750
+
723
751
  class AuthGate():
724
752
  '''
725
753
  The storage object for object specific rules for users/roles.
@@ -975,7 +1003,7 @@ class User(Ruler):
975
1003
  if nexs:
976
1004
  return await self.auth.setUserInfo(self.iden, name, valu, gateiden=gateiden, mesg=mesg)
977
1005
  else:
978
- return await self.auth._hndlsetUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
1006
+ return await self.auth._setUserInfo(self.iden, name, valu, gateiden=gateiden, logged=nexs, mesg=mesg)
979
1007
 
980
1008
  async def setName(self, name):
981
1009
  return await self.auth.setUserName(self.iden, name)
@@ -1289,7 +1317,7 @@ class User(Ruler):
1289
1317
  if nexs:
1290
1318
  await self.auth.setUserInfo(self.iden, 'roles', roles, mesg=mesg)
1291
1319
  else:
1292
- await self.auth._hndlsetUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
1320
+ await self.auth._setUserInfo(self.iden, 'roles', roles, logged=nexs, mesg=mesg)
1293
1321
 
1294
1322
  def isLocked(self):
1295
1323
  return self.info.get('locked')
@@ -1332,7 +1360,7 @@ class User(Ruler):
1332
1360
  if logged:
1333
1361
  await self.auth.setUserInfo(self.iden, 'admin', admin, gateiden=gateiden)
1334
1362
  else:
1335
- await self.auth._hndlsetUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
1363
+ await self.auth._setUserInfo(self.iden, 'admin', admin, gateiden=gateiden, logged=logged)
1336
1364
 
1337
1365
  async def setLocked(self, locked, logged=True):
1338
1366
  if not isinstance(locked, bool):
@@ -1352,9 +1380,9 @@ class User(Ruler):
1352
1380
  await self.auth.setUserInfo(self.iden, 'policy:attempts', 0)
1353
1381
 
1354
1382
  else:
1355
- await self.auth._hndlsetUserInfo(self.iden, 'locked', locked, logged=logged)
1383
+ await self.auth._setUserInfo(self.iden, 'locked', locked, logged=logged)
1356
1384
  if resetAttempts:
1357
- await self.auth._hndlsetUserInfo(self.iden, 'policy:attempts', 0)
1385
+ await self.auth._setUserInfo(self.iden, 'policy:attempts', 0)
1358
1386
 
1359
1387
  async def setArchived(self, archived):
1360
1388
  if not isinstance(archived, bool):
@@ -1541,7 +1569,7 @@ class User(Ruler):
1541
1569
  if nexs:
1542
1570
  await self.auth.setUserInfo(self.iden, 'policy:previous', previous[:prevvalu])
1543
1571
  else:
1544
- await self.auth._hndlsetUserInfo(self.iden, 'policy:previous', previous[:prevvalu], logged=nexs)
1572
+ await self.auth._setUserInfo(self.iden, 'policy:previous', previous[:prevvalu], logged=nexs)
1545
1573
 
1546
1574
  async def setPasswd(self, passwd, nexs=True, enforce_policy=True):
1547
1575
  # Prevent empty string or non-string values
@@ -1559,4 +1587,4 @@ class User(Ruler):
1559
1587
  if nexs:
1560
1588
  await self.auth.setUserInfo(self.iden, 'passwd', shadow)
1561
1589
  else:
1562
- await self.auth._hndlsetUserInfo(self.iden, 'passwd', shadow, logged=nexs)
1590
+ await self.auth._setUserInfo(self.iden, 'passwd', shadow, logged=nexs)
synapse/lib/cell.py CHANGED
@@ -62,7 +62,7 @@ import synapse.tools.backup as s_t_backup
62
62
 
63
63
  logger = logging.getLogger(__name__)
64
64
 
65
- NEXUS_VERSION = (2, 177)
65
+ NEXUS_VERSION = (2, 198)
66
66
 
67
67
  SLAB_MAP_SIZE = 128 * s_const.mebibyte
68
68
  SSLCTX_CACHE_SIZE = 64
@@ -1625,16 +1625,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1625
1625
  pass
1626
1626
 
1627
1627
  async def setNexsVers(self, vers):
1628
- if self.nexsvers < NEXUS_VERSION:
1629
- await self._push('nexs:vers:set', NEXUS_VERSION)
1628
+ if self.nexsvers < vers:
1629
+ await self._push('nexs:vers:set', vers)
1630
1630
 
1631
1631
  @s_nexus.Pusher.onPush('nexs:vers:set')
1632
1632
  async def _setNexsVers(self, vers):
1633
1633
  if vers > self.nexsvers:
1634
- self.cellvers.set('nexus:version', vers)
1634
+ await self._migrNexsVers(vers)
1635
+ self.cellinfo.set('nexus:version', vers)
1635
1636
  self.nexsvers = vers
1636
1637
  await self.configNexsVers()
1637
1638
 
1639
+ async def _migrNexsVers(self, newvers):
1640
+ if self.nexsvers < (2, 198) and newvers >= (2, 198) and self.conf.get('auth:ctor') is None:
1641
+ # This "migration" will lock all archived users. Once the nexus version is bumped to
1642
+ # >=2.198, then the bottom-half nexus handler for user:info (Auth._setUserInfo()) will
1643
+ # begin rejecting unlock requests for archived users.
1644
+
1645
+ authkv = self.slab.getSafeKeyVal('auth')
1646
+ userkv = authkv.getSubKeyVal('user:info:')
1647
+
1648
+ for iden, info in userkv.items():
1649
+ if info.get('archived') and not info.get('locked'):
1650
+ info['locked'] = True
1651
+ userkv.set(iden, info)
1652
+
1653
+ # Clear the auth caches so the changes get picked up by the already running auth subsystem
1654
+ self.auth.clearAuthCache()
1655
+
1638
1656
  async def configNexsVers(self):
1639
1657
  for meth, orig in self.nexspatches:
1640
1658
  setattr(self, meth, orig)
synapse/lib/hive.py CHANGED
@@ -424,7 +424,7 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
424
424
  return node.valu
425
425
 
426
426
  async def getTeleApi(self, link, mesg, path):
427
-
427
+ s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
428
428
  auth = await self.getHiveAuth()
429
429
 
430
430
  if not self.conf.get('auth:en'):
@@ -759,6 +759,7 @@ def iterpath(path):
759
759
  yield path[:i + 1]
760
760
 
761
761
  async def openurl(url, **opts):
762
+ s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
762
763
  prox = await s_telepath.openurl(url, **opts)
763
764
  return await TeleHive.anit(prox)
764
765
 
synapse/lib/hiveauth.py CHANGED
@@ -1,3 +1,9 @@
1
+ # pragma: no cover
2
+
3
+ ###
4
+ ### THIS WHOLE MODULE IS DEPRECATED AND EXPECTED TO BE REMOVED O/A v2.198.0
5
+ ###
6
+
1
7
  import logging
2
8
  import dataclasses
3
9
 
@@ -30,12 +36,13 @@ reqValidRules = s_config.getJsValidator({
30
36
 
31
37
  def getShadow(passwd): # pragma: no cover
32
38
  '''This API is deprecated.'''
33
- s_common.deprecated('hiveauth.getShadow()', curv='2.110.0')
39
+ s_common.deprecated('hiveauth.getShadow()', curv='2.110.0', eolv='2.199.0')
34
40
  salt = s_common.guid()
35
41
  hashed = s_common.guid((salt, passwd))
36
42
  return (salt, hashed)
37
43
 
38
44
  def textFromRule(rule):
45
+ s_common.deprecated('hiveauth.textFromRule()', curv='2.198.0', eolv='2.199.0') # pragma: no cover
39
46
  text = '.'.join(rule[1])
40
47
  if not rule[0]:
41
48
  text = '!' + text
@@ -43,6 +50,7 @@ def textFromRule(rule):
43
50
 
44
51
  @dataclasses.dataclass(slots=True)
45
52
  class _allowedReason:
53
+ s_common.deprecated('hiveauth._allowedReason()', curv='2.198.0', eolv='2.199.0')
46
54
  value: Union[bool | None]
47
55
  default: bool = False
48
56
  isadmin: bool = False
@@ -135,6 +143,7 @@ class Auth(s_nexus.Pusher):
135
143
  Args:
136
144
  node (HiveNode): The root of the persistent storage for auth
137
145
  '''
146
+ s_common.deprecated('Auth.__anit__()', curv='2.198.0', eolv='2.199.0')
138
147
  # Derive an iden from the parent
139
148
  iden = 'auth:' + ':'.join(node.full)
140
149
  await s_nexus.Pusher.__anit__(self, iden, nexsroot=nexsroot)
synapse/lib/jsonstor.py CHANGED
@@ -46,7 +46,7 @@ class JsonStor(s_base.Base):
46
46
  self.dirty.pop(buid, None)
47
47
  await asyncio.sleep(0)
48
48
 
49
- def _incRefObj(self, buid, valu=1):
49
+ async def _incRefObj(self, buid, valu=1):
50
50
 
51
51
  refs = 0
52
52
 
@@ -62,6 +62,7 @@ class JsonStor(s_base.Base):
62
62
  # remove the meta entries
63
63
  for lkey, byts in self.slab.scanByPref(buid, db=self.metadb):
64
64
  self.slab.pop(lkey, db=self.metadb)
65
+ await asyncio.sleep(0)
65
66
 
66
67
  # remove the item data
67
68
  self.slab.pop(buid, db=self.itemdb)
@@ -88,7 +89,7 @@ class JsonStor(s_base.Base):
88
89
 
89
90
  oldb = self.slab.replace(pkey, buid, db=self.pathdb)
90
91
  if oldb is not None:
91
- self._incRefObj(oldb, -1)
92
+ await self._incRefObj(oldb, -1)
92
93
 
93
94
  self.slab.put(buid + b'refs', s_msgpack.en(1), db=self.metadb)
94
95
 
@@ -124,7 +125,7 @@ class JsonStor(s_base.Base):
124
125
  pkey = self._pathToPkey(path)
125
126
  buid = self.slab.pop(pkey, db=self.pathdb)
126
127
  if buid is not None:
127
- self._incRefObj(buid, valu=-1)
128
+ await self._incRefObj(buid, valu=-1)
128
129
 
129
130
  async def setPathLink(self, srcpath, dstpath):
130
131
  '''
@@ -140,9 +141,9 @@ class JsonStor(s_base.Base):
140
141
 
141
142
  oldb = self.slab.pop(srcpkey, db=self.pathdb)
142
143
  if oldb is not None:
143
- self._incRefObj(oldb, valu=-1)
144
+ await self._incRefObj(oldb, valu=-1)
144
145
 
145
- self._incRefObj(buid, valu=1)
146
+ await self._incRefObj(buid, valu=1)
146
147
  self.slab.put(srcpkey, buid, db=self.pathdb)
147
148
 
148
149
  async def getPathObjProp(self, path, prop):
synapse/lib/layer.py CHANGED
@@ -3543,9 +3543,8 @@ class Layer(s_nexus.Pusher):
3543
3543
  if self.nodeDelHook is not None:
3544
3544
  self.nodeDelHook()
3545
3545
 
3546
- self._wipeNodeData(buid)
3547
- # TODO edits to become async so we can sleep(0) on large deletes?
3548
- self._delNodeEdges(buid)
3546
+ await self._wipeNodeData(buid)
3547
+ await self._delNodeEdges(buid)
3549
3548
 
3550
3549
  self.buidcache.pop(buid, None)
3551
3550
 
@@ -3957,13 +3956,14 @@ class Layer(s_nexus.Pusher):
3957
3956
  for _, lval in self.layrslab.scanByDups(verb.encode(), db=self.byverb):
3958
3957
  yield (s_common.ehex(lval[:32]), verb, s_common.ehex(lval[32:]))
3959
3958
 
3960
- def _delNodeEdges(self, buid):
3959
+ async def _delNodeEdges(self, buid):
3961
3960
  for lkey, n2buid in self.layrslab.scanByPref(buid, db=self.edgesn1):
3962
3961
  venc = lkey[32:]
3963
3962
  self.layrslab.delete(venc, buid + n2buid, db=self.byverb)
3964
3963
  self.layrslab.delete(lkey, n2buid, db=self.edgesn1)
3965
3964
  self.layrslab.delete(n2buid + venc, buid, db=self.edgesn2)
3966
3965
  self.layrslab.delete(buid + n2buid, venc, db=self.edgesn1n2)
3966
+ await asyncio.sleep(0)
3967
3967
 
3968
3968
  def getStorIndx(self, stortype, valu):
3969
3969
 
@@ -4473,7 +4473,7 @@ class Layer(s_nexus.Pusher):
4473
4473
 
4474
4474
  await self.waitfini(1)
4475
4475
 
4476
- def _wipeNodeData(self, buid):
4476
+ async def _wipeNodeData(self, buid):
4477
4477
  '''
4478
4478
  Remove all node data for a buid
4479
4479
  '''
@@ -4482,6 +4482,7 @@ class Layer(s_nexus.Pusher):
4482
4482
  buid = lkey[:32]
4483
4483
  self.dataslab.delete(lkey, db=self.nodedata)
4484
4484
  self.dataslab.delete(abrv, buid, db=self.dataname)
4485
+ await asyncio.sleep(0)
4485
4486
 
4486
4487
  async def getModelVers(self):
4487
4488
  return self.layrinfo.get('model:version', (-1, -1, -1))
synapse/lib/node.py CHANGED
@@ -63,18 +63,24 @@ class Node:
63
63
  def __repr__(self):
64
64
  return f'Node{{{self.pack()}}}'
65
65
 
66
- async def addEdge(self, verb, n2iden):
66
+ async def addEdge(self, verb, n2iden, extra=None):
67
67
  if self.form.isrunt:
68
68
  mesg = f'Edges cannot be used with runt nodes: {self.form.full}'
69
- raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
69
+ exc = s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
70
+ if extra is not None:
71
+ exc = extra(exc)
72
+ raise exc
70
73
 
71
74
  async with self.snap.getNodeEditor(self) as editor:
72
75
  return await editor.addEdge(verb, n2iden)
73
76
 
74
- async def delEdge(self, verb, n2iden):
77
+ async def delEdge(self, verb, n2iden, extra=None):
75
78
  if self.form.isrunt:
76
79
  mesg = f'Edges cannot be used with runt nodes: {self.form.full}'
77
- raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
80
+ exc = s_exc.IsRuntForm(mesg=mesg, form=self.form.full)
81
+ if extra is not None:
82
+ exc = extra(exc)
83
+ raise exc
78
84
 
79
85
  async with self.snap.getNodeEditor(self) as editor:
80
86
  return await editor.delEdge(verb, n2iden)
synapse/lib/parser.py CHANGED
@@ -6,6 +6,7 @@ import lark # type: ignore
6
6
  import regex # type: ignore
7
7
 
8
8
  import synapse.exc as s_exc
9
+ import synapse.common as s_common
9
10
 
10
11
  import synapse.lib.ast as s_ast
11
12
  import synapse.lib.coro as s_coro
@@ -72,6 +73,7 @@ terminalEnglishMap = {
72
73
  'LSQB': '[',
73
74
  'MCASEBARE': 'case multi-value',
74
75
  'MODSET': '+= or -=',
76
+ 'MODSETMULTI': '++= or --=',
75
77
  'NONQUOTEWORD': 'unquoted value',
76
78
  'NOT': 'not',
77
79
  'NULL': 'null',
@@ -92,8 +94,8 @@ terminalEnglishMap = {
92
94
  'TRY': 'try',
93
95
  'TRIPLEQUOTEDSTRING': 'triple-quoted string',
94
96
  'TRYSET': '?=',
95
- 'TRYSETPLUS': '?+=',
96
- 'TRYSETMINUS': '?-=',
97
+ 'TRYMODSET': '?+= or ?-=',
98
+ 'TRYMODSETMULTI': '?++= or ?--=',
97
99
  'UNIVNAME': 'universal property',
98
100
  'UNSET': 'unset',
99
101
  'EXPRUNIVNAME': 'universal property',
@@ -158,7 +160,7 @@ class AstConverter(lark.Transformer):
158
160
 
159
161
  # Keep the original text for error printing and weird subquery argv parsing
160
162
  self.text = text
161
- self.texthash = hashlib.md5(text.encode(errors='surrogatepass'), usedforsecurity=False).hexdigest()
163
+ self.texthash = s_common.queryhash(text)
162
164
 
163
165
  def metaToAstInfo(self, meta, isterm=False):
164
166
  if isinstance(meta, lark.tree.Meta) and meta.empty:
@@ -248,9 +250,14 @@ class AstConverter(lark.Transformer):
248
250
  @lark.v_args(meta=True)
249
251
  def embedquery(self, meta, kids):
250
252
  assert len(kids) == 1
251
- astinfo = self.metaToAstInfo(meta)
252
- text = kids[0].getAstText()
253
- return s_ast.EmbedQuery(kids[0].astinfo, text, kids=kids)
253
+ astinfo = AstInfo(self.text,
254
+ meta.start_pos + 2, meta.end_pos - 1,
255
+ meta.line, meta.end_line,
256
+ meta.column, meta.end_column, False)
257
+
258
+ kids[0].astinfo = astinfo
259
+
260
+ return s_ast.EmbedQuery(astinfo, kids[0].getAstText(), kids=kids)
254
261
 
255
262
  @lark.v_args(meta=True)
256
263
  def funccall(self, meta, kids):
@@ -489,18 +496,28 @@ class Parser:
489
496
  Convert lark exception to synapse BadSyntax exception
490
497
  '''
491
498
  mesg = regex.split('[\n]', str(e))[0]
492
- at = len(self.text)
493
- line = None
494
- column = None
499
+ soff = eoff = len(self.text)
500
+ sline = eline = None
501
+ scol = ecol = None
495
502
  token = None
496
503
  if isinstance(e, lark.exceptions.UnexpectedToken):
497
504
  expected = sorted(set(terminalEnglishMap[t] for t in e.expected))
498
- at = e.pos_in_stream
499
- line = e.line
500
- column = e.column
501
505
  token = e.token.value
502
- valu = terminalEnglishMap.get(e.token.type, e.token.value)
503
- mesg = f"Unexpected token '{valu}' at line {line}, column {column}," \
506
+ soff = e.pos_in_stream
507
+ eoff = soff + len(token)
508
+
509
+ lines = token.splitlines()
510
+ sline = e.line
511
+ eline = sline + len(lines) - 1
512
+
513
+ scol = e.column
514
+ if len(lines) > 1:
515
+ ecol = len(lines[-1])
516
+ else:
517
+ ecol = scol + len(token)
518
+
519
+ valu = terminalEnglishMap.get(e.token.type, token)
520
+ mesg = f"Unexpected token '{valu}' at line {sline}, column {scol}," \
504
521
  f' expecting one of: {", ".join(expected)}'
505
522
 
506
523
  elif isinstance(e, lark.exceptions.VisitError):
@@ -514,16 +531,23 @@ class Parser:
514
531
  elif isinstance(e, lark.exceptions.UnexpectedCharacters): # pragma: no cover
515
532
  expected = sorted(set(terminalEnglishMap[t] for t in e.allowed))
516
533
  mesg += f'. Expecting one of: {", ".join(expected)}'
517
- at = e.pos_in_stream
518
- line = e.line
519
- column = e.column
534
+ soff = eoff = e.pos_in_stream
535
+ sline = eline = e.line
536
+ scol = ecol = e.column
520
537
  elif isinstance(e, lark.exceptions.UnexpectedEOF): # pragma: no cover
521
538
  expected = sorted(set(terminalEnglishMap[t] for t in e.expected))
522
539
  mesg += ' ' + ', '.join(expected)
523
- line = e.line
524
- column = e.column
525
-
526
- return s_exc.BadSyntax(at=at, text=self.text, mesg=mesg, line=line, column=column, token=token)
540
+ sline = eline = e.line
541
+ scol = ecol = e.column
542
+
543
+ highlight = {
544
+ 'hash': s_common.queryhash(self.text),
545
+ 'lines': (sline, eline),
546
+ 'columns': (scol, ecol),
547
+ 'offsets': (soff, eoff),
548
+ }
549
+ return s_exc.BadSyntax(at=soff, text=self.text, mesg=mesg, line=sline,
550
+ column=scol, token=token, highlight=highlight)
527
551
 
528
552
  def eval(self):
529
553
  try:
@@ -661,6 +685,7 @@ ruleClassMap = {
661
685
  'editpropdel': lambda astinfo, kids: s_ast.EditPropDel(astinfo, kids[1:]),
662
686
  'editpropset': s_ast.EditPropSet,
663
687
  'editcondpropset': s_ast.EditCondPropSet,
688
+ 'editpropsetmulti': s_ast.EditPropSetMulti,
664
689
  'edittagadd': s_ast.EditTagAdd,
665
690
  'edittagdel': lambda astinfo, kids: s_ast.EditTagDel(astinfo, kids[1:]),
666
691
  'edittagpropset': s_ast.EditTagPropSet,
synapse/lib/schemas.py CHANGED
@@ -880,6 +880,19 @@ _reqValidPkgdefSchema = {
880
880
  'type': 'string',
881
881
  'pattern': s_grammar.re_scmd
882
882
  },
883
+ 'endpoints': {
884
+ 'type': 'array',
885
+ 'items': {
886
+ 'type': 'object',
887
+ 'properties': {
888
+ 'path': {'type': 'string'},
889
+ 'host': {'type': 'string'},
890
+ 'desc': {'type': 'string'},
891
+ },
892
+ 'required': ['path'],
893
+ 'additionalProperties': False
894
+ }
895
+ },
883
896
  'cmdargs': {
884
897
  'type': ['array', 'null'],
885
898
  'items': {'$ref': '#/definitions/cmdarg'},
synapse/lib/snap.py CHANGED
@@ -138,6 +138,12 @@ class ProtoNode:
138
138
 
139
139
  if not await self.ctx.snap.hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
140
140
  self.edges.add(tupl)
141
+
142
+ if len(self.edges) >= 1000:
143
+ nodeedits = self.ctx.getNodeEdits()
144
+ await self.ctx.snap.applyNodeEdits(nodeedits)
145
+ self.edges.clear()
146
+
141
147
  return True
142
148
 
143
149
  return False
@@ -169,6 +175,12 @@ class ProtoNode:
169
175
 
170
176
  if await self.ctx.snap.layers[-1].hasNodeEdge(self.buid, verb, s_common.uhex(n2iden)):
171
177
  self.edgedels.add(tupl)
178
+
179
+ if len(self.edgedels) >= 1000:
180
+ nodeedits = self.ctx.getNodeEdits()
181
+ await self.ctx.snap.applyNodeEdits(nodeedits)
182
+ self.edgedels.clear()
183
+
172
184
  return True
173
185
 
174
186
  return False
@@ -1062,6 +1074,23 @@ class Snap(s_base.Base):
1062
1074
  mesg = f'No property named "{full}".'
1063
1075
  raise s_exc.NoSuchProp(mesg=mesg)
1064
1076
 
1077
+ if isinstance(valu, dict) and isinstance(prop.type, s_types.Guid) and cmpr == '=':
1078
+ if prop.isform:
1079
+ if (node := await self._getGuidNodeByDict(prop, valu)) is not None:
1080
+ yield node
1081
+ return
1082
+
1083
+ fname = prop.type.name
1084
+ if (form := prop.modl.form(fname)) is None:
1085
+ mesg = f'The property "{full}" type "{fname}" is not a form and cannot be lifted using a dictionary.'
1086
+ raise s_exc.BadTypeValu(mesg=mesg)
1087
+
1088
+ if (node := await self._getGuidNodeByDict(form, valu)) is None:
1089
+ return
1090
+
1091
+ norm = False
1092
+ valu = node.ndef[1]
1093
+
1065
1094
  if norm:
1066
1095
  cmprvals = prop.type.getStorCmprs(cmpr, valu)
1067
1096
  # an empty return probably means ?= with invalid value
@@ -1398,10 +1427,6 @@ class Snap(s_base.Base):
1398
1427
 
1399
1428
  async def _addGuidNodeByDict(self, form, vals, props=None):
1400
1429
 
1401
- norms = {}
1402
- counts = []
1403
- proplist = []
1404
-
1405
1430
  if props is None:
1406
1431
  props = {}
1407
1432
 
@@ -1439,7 +1464,32 @@ class Snap(s_base.Base):
1439
1464
  raise e
1440
1465
  await self.warn(f'Skipping bad value for prop {form.name}:{name}: {mesg}')
1441
1466
 
1442
- for name, valu in vals.items():
1467
+ norms, proplist = self._normGuidNodeDict(form, vals)
1468
+
1469
+ iden = s_common.guid(proplist)
1470
+ node = await self._getGuidNodeByNorms(form, iden, norms)
1471
+
1472
+ async with self.getEditor() as editor:
1473
+
1474
+ if node is not None:
1475
+ proto = editor.loadNode(node)
1476
+ else:
1477
+ proto = await editor.addNode(form.name, iden)
1478
+ for name, (prop, valu, info) in norms.items():
1479
+ await proto.set(name, valu, norminfo=info)
1480
+
1481
+ # ensure the non-deconf props are set
1482
+ for name, (valu, info) in props.items():
1483
+ await proto.set(name, valu, norminfo=info)
1484
+
1485
+ return await self.getNodeByBuid(proto.buid)
1486
+
1487
+ def _normGuidNodeDict(self, form, props):
1488
+
1489
+ norms = {}
1490
+ proplist = []
1491
+
1492
+ for name, valu in props.items():
1443
1493
 
1444
1494
  try:
1445
1495
  prop = form.reqProp(name)
@@ -1458,22 +1508,24 @@ class Snap(s_base.Base):
1458
1508
 
1459
1509
  proplist.sort()
1460
1510
 
1511
+ return norms, proplist
1512
+
1513
+ async def _getGuidNodeByDict(self, form, props):
1514
+ norms, proplist = self._normGuidNodeDict(form, props)
1515
+ return await self._getGuidNodeByNorms(form, s_common.guid(proplist), norms)
1516
+
1517
+ async def _getGuidNodeByNorms(self, form, iden, norms):
1518
+
1461
1519
  # check first for an exact match via our same deconf strategy
1462
- iden = s_common.guid(proplist)
1463
1520
 
1464
1521
  node = await self.getNodeByNdef((form.full, iden))
1465
1522
  if node is not None:
1466
1523
 
1467
1524
  # ensure we still match the property deconf criteria
1468
- for name, (prop, norm, info) in norms.items():
1525
+ for (prop, norm, info) in norms.values():
1469
1526
  if not self._filtByPropAlts(node, prop, norm):
1470
1527
  break
1471
1528
  else:
1472
- # ensure the non-deconf props are set
1473
- async with self.getEditor() as editor:
1474
- proto = editor.loadNode(node)
1475
- for name, (valu, info) in props.items():
1476
- await proto.set(name, valu, norminfo=info)
1477
1529
  return node
1478
1530
 
1479
1531
  # TODO there is an opportunity here to populate
@@ -1482,8 +1534,10 @@ class Snap(s_base.Base):
1482
1534
  # if we lookup a node and it no longer passes the
1483
1535
  # filter...
1484
1536
 
1537
+ counts = []
1538
+
1485
1539
  # no exact match. lets do some counting.
1486
- for name, (prop, norm, info) in norms.items():
1540
+ for (prop, norm, info) in norms.values():
1487
1541
  count = await self._getPropAltCount(prop, norm)
1488
1542
  counts.append((count, prop, norm))
1489
1543
 
@@ -1499,21 +1553,9 @@ class Snap(s_base.Base):
1499
1553
  if not self._filtByPropAlts(node, prop, norm):
1500
1554
  break
1501
1555
  else:
1502
- # ensure the non-deconf props are set
1503
- async with self.getEditor() as editor:
1504
- proto = editor.loadNode(node)
1505
- for name, (valu, info) in props.items():
1506
- await proto.set(name, valu, norminfo=info)
1507
1556
  return node
1508
1557
 
1509
- async with self.getEditor() as editor:
1510
- proto = await editor.addNode(form.name, iden)
1511
- for name, (prop, valu, info) in norms.items():
1512
- await proto.set(name, valu, norminfo=info)
1513
- for name, (valu, info) in props.items():
1514
- await proto.set(name, valu, norminfo=info)
1515
-
1516
- return await self.getNodeByBuid(proto.buid)
1558
+ return None
1517
1559
 
1518
1560
  async def _getPropAltCount(self, prop, valu):
1519
1561
  count = 0