synapse 2.212.0__py311-none-any.whl → 2.214.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (75) hide show
  1. synapse/cortex.py +37 -6
  2. synapse/daemon.py +6 -6
  3. synapse/exc.py +13 -1
  4. synapse/lib/aha.py +5 -0
  5. synapse/lib/ast.py +2 -6
  6. synapse/lib/boss.py +47 -2
  7. synapse/lib/cell.py +199 -6
  8. synapse/lib/certdir.py +44 -1
  9. synapse/lib/cmd.py +24 -0
  10. synapse/lib/coro.py +8 -2
  11. synapse/lib/drive.py +7 -2
  12. synapse/lib/link.py +11 -3
  13. synapse/lib/schemas.py +1 -1
  14. synapse/lib/scrape.py +3 -1
  15. synapse/lib/snap.py +89 -80
  16. synapse/lib/storm.py +2 -1
  17. synapse/lib/stormlib/imap.py +3 -2
  18. synapse/lib/stormlib/spooled.py +4 -0
  19. synapse/lib/stormtypes.py +18 -0
  20. synapse/lib/task.py +1 -0
  21. synapse/lib/types.py +36 -8
  22. synapse/lib/version.py +2 -2
  23. synapse/models/inet.py +5 -0
  24. synapse/telepath.py +4 -2
  25. synapse/tests/files/testpkg_build_docs/docs/bar.rst +15 -0
  26. synapse/tests/files/testpkg_build_docs/docs/foo.rst +4 -0
  27. synapse/tests/files/testpkg_build_docs/storm/commands/testcmd.storm +0 -0
  28. synapse/tests/files/testpkg_build_docs/storm/modules/apimod.storm +0 -0
  29. synapse/tests/files/testpkg_build_docs/storm/modules/testmod.storm +0 -0
  30. synapse/tests/files/testpkg_build_docs/storm/testcmd.storm +5 -0
  31. synapse/tests/files/testpkg_build_docs/testpkg.yaml +69 -0
  32. synapse/tests/test_cortex.py +20 -1
  33. synapse/tests/test_daemon.py +1 -1
  34. synapse/tests/test_exc.py +6 -0
  35. synapse/tests/test_lib_ast.py +69 -14
  36. synapse/tests/test_lib_boss.py +8 -0
  37. synapse/tests/test_lib_cell.py +119 -8
  38. synapse/tests/test_lib_certdir.py +8 -0
  39. synapse/tests/test_lib_coro.py +5 -0
  40. synapse/tests/test_lib_httpapi.py +10 -2
  41. synapse/tests/test_lib_link.py +1 -1
  42. synapse/tests/test_lib_scrape.py +6 -0
  43. synapse/tests/test_lib_storm.py +123 -1
  44. synapse/tests/test_lib_stormlib_spooled.py +31 -0
  45. synapse/tests/test_lib_stormtypes.py +11 -0
  46. synapse/tests/test_lib_types.py +137 -45
  47. synapse/tests/test_model_crypto.py +8 -0
  48. synapse/tests/test_model_inet.py +7 -0
  49. synapse/tests/test_telepath.py +50 -5
  50. synapse/tests/test_tools_axon.py +304 -0
  51. synapse/tests/test_tools_cortex_layer.py +419 -0
  52. synapse/tests/test_tools_demote.py +114 -0
  53. synapse/tests/test_tools_pkgs_gendocs.py +100 -0
  54. synapse/tests/test_tools_shutdown.py +95 -0
  55. synapse/tests/test_utils.py +22 -1
  56. synapse/tests/utils.py +44 -29
  57. synapse/tools/aha/easycert.py +2 -0
  58. synapse/tools/aha/enroll.py +3 -0
  59. synapse/tools/axon/__init__.py +0 -0
  60. synapse/tools/axon/dump.py +155 -0
  61. synapse/tools/axon/load.py +89 -0
  62. synapse/tools/cortex/__init__.py +0 -0
  63. synapse/tools/cortex/layer/__init__.py +0 -0
  64. synapse/tools/cortex/layer/dump.py +184 -0
  65. synapse/tools/cortex/layer/load.py +129 -0
  66. synapse/tools/demote.py +52 -0
  67. synapse/tools/healthcheck.py +1 -1
  68. synapse/tools/pkgs/gendocs.py +176 -0
  69. synapse/tools/pkgs/pandoc_filter.py +79 -0
  70. synapse/tools/shutdown.py +52 -0
  71. {synapse-2.212.0.dist-info → synapse-2.214.0.dist-info}/METADATA +1 -1
  72. {synapse-2.212.0.dist-info → synapse-2.214.0.dist-info}/RECORD +75 -52
  73. {synapse-2.212.0.dist-info → synapse-2.214.0.dist-info}/WHEEL +0 -0
  74. {synapse-2.212.0.dist-info → synapse-2.214.0.dist-info}/licenses/LICENSE +0 -0
  75. {synapse-2.212.0.dist-info → synapse-2.214.0.dist-info}/top_level.txt +0 -0
synapse/lib/types.py CHANGED
@@ -651,9 +651,10 @@ class Hex(Type):
651
651
 
652
652
  def postTypeInit(self):
653
653
  self._size = self.opts.get('size')
654
+ self._zeropad = self.opts.get('zeropad')
654
655
 
655
656
  # This is for backward compat with v2.142.x where zeropad was a bool
656
- self._zeropad = self.opts.get('zeropad')
657
+ # TODO: Remove this compat check in 3xx
657
658
  if isinstance(self._zeropad, bool):
658
659
  if self._zeropad:
659
660
  self._zeropad = self._size
@@ -678,6 +679,7 @@ class Hex(Type):
678
679
  if self._size:
679
680
  self._zeropad = min(self._zeropad, self._size)
680
681
 
682
+ self.setNormFunc(int, self._normPyInt)
681
683
  self.setNormFunc(str, self._normPyStr)
682
684
  self.setNormFunc(bytes, self._normPyBytes)
683
685
  self.storlifts.update({
@@ -703,21 +705,47 @@ class Hex(Type):
703
705
  return self._storLiftNorm(cmpr, valu)
704
706
 
705
707
  def _storLiftPref(self, cmpr, valu):
708
+ if not isinstance(valu, str):
709
+ vtyp = type(valu).__name__
710
+ mesg = f'Hex prefix lift values must be str, not {vtyp}.'
711
+ raise s_exc.BadTypeValu(mesg=mesg, type=vtyp, name=self.name)
712
+
706
713
  valu = self._preNormHex(valu)
707
714
  return (
708
715
  ('^=', valu, self.stortype),
709
716
  )
710
717
 
718
+ def _normPyInt(self, valu):
719
+ extra = 7
720
+ if valu < 0:
721
+ # Negative values need a little more space to store the sign
722
+ extra = 8
723
+
724
+ bytelen = max((valu.bit_length() + extra) // 8, self._zeropad // 2)
725
+
726
+ try:
727
+ byts = valu.to_bytes(bytelen, 'big', signed=(valu < 0))
728
+ hexval = s_common.ehex(byts)
729
+
730
+ except OverflowError as e: # pragma: no cover
731
+ mesg = f'Invalid width for {valu}.'
732
+ raise s_exc.BadTypeValu(mesg=mesg, name=self.name)
733
+
734
+ if self._size and len(hexval) != self._size:
735
+ raise s_exc.BadTypeValu(valu=valu, reqwidth=self._size, name=self.name,
736
+ mesg='Invalid width.')
737
+
738
+ return hexval, {}
739
+
711
740
  def _normPyStr(self, valu):
712
- valu = valu.strip().lower()
713
- if valu.startswith('0x'):
714
- valu = valu[2:]
741
+ valu = self._preNormHex(valu)
715
742
 
716
- valu = valu.replace(' ', '').replace(':', '')
743
+ if len(valu) % 2 != 0:
744
+ valu = f'0{valu}'
717
745
 
718
746
  if not valu:
719
- raise s_exc.BadTypeValu(valu=valu, name='hex',
720
- mesg='No string left after stripping')
747
+ raise s_exc.BadTypeValu(valu=valu, name=self.name,
748
+ mesg='No string left after stripping.')
721
749
 
722
750
  if self._zeropad and len(valu) < self._zeropad:
723
751
  padlen = self._zeropad - len(valu)
@@ -732,7 +760,7 @@ class Hex(Type):
732
760
 
733
761
  if self._size and len(valu) != self._size:
734
762
  raise s_exc.BadTypeValu(valu=valu, reqwidth=self._size, name=self.name,
735
- mesg='invalid width')
763
+ mesg='Invalid width.')
736
764
  return valu, {}
737
765
 
738
766
  def _normPyBytes(self, valu):
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 212, 0)
226
+ version = (2, 214, 0)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = '68ec8184b0e2469fd1ef30cbeb29f5989c60c768'
228
+ commit = '41e2ed0599236256e9452b8afeb152660e68e2c2'
synapse/models/inet.py CHANGED
@@ -1780,6 +1780,8 @@ class InetModule(s_module.CoreModule):
1780
1780
  ('remover', ('inet:service:account', {}), {
1781
1781
  'doc': 'The service account which removed or decommissioned the {service:base}.'}),
1782
1782
 
1783
+ ('app', ('inet:service:app', {}), {
1784
+ 'doc': 'The app which contains the {service:base}.'}),
1783
1785
  ),
1784
1786
  }),
1785
1787
 
@@ -3748,6 +3750,9 @@ class InetModule(s_module.CoreModule):
3748
3750
 
3749
3751
  ('tenant', ('inet:service:tenant', {}), {
3750
3752
  'doc': 'The tenant which contains the instance.'}),
3753
+
3754
+ ('app', ('inet:service:app', {}), {
3755
+ 'doc': 'The app which contains the instance.'}),
3751
3756
  )),
3752
3757
 
3753
3758
  ('inet:service:app', {}, (
synapse/telepath.py CHANGED
@@ -1605,8 +1605,6 @@ async def openinfo(info):
1605
1605
  # cell://rel/path/to/celldir:share
1606
1606
  path = info.get('path')
1607
1607
 
1608
- name = info.get('name', '*')
1609
-
1610
1608
  # support cell://<relpath>/<to>/<cell>
1611
1609
  # by detecting host...
1612
1610
  host = info.get('host')
@@ -1614,9 +1612,12 @@ async def openinfo(info):
1614
1612
  path = path.strip('/')
1615
1613
  path = os.path.join(host, path)
1616
1614
 
1615
+ name = '*'
1617
1616
  if ':' in path:
1618
1617
  path, name = path.split(':')
1619
1618
 
1619
+ name = info.get('name', name)
1620
+
1620
1621
  full = os.path.join(path, 'sock')
1621
1622
  link = await s_link.unixconnect(full)
1622
1623
 
@@ -1626,6 +1627,7 @@ async def openinfo(info):
1626
1627
  path = info.get('path')
1627
1628
  if ':' in path:
1628
1629
  path, name = path.split(':')
1630
+ name = info.get('name', name)
1629
1631
  link = await s_link.unixconnect(path)
1630
1632
 
1631
1633
  elif scheme in ('tcp', 'ssl'):
@@ -0,0 +1,15 @@
1
+ :tocdepth: 2
2
+
3
+ :orphan:
4
+
5
+ .. highlight:: none
6
+ .. storm-cortex:: default
7
+
8
+ Bar
9
+ ---
10
+
11
+ A wild node appears!
12
+
13
+ .. storm-cli:: [inet:asn=1]
14
+
15
+ Words!
@@ -0,0 +1,4 @@
1
+ Hello world
2
+ -----------
3
+
4
+ Words
@@ -0,0 +1,5 @@
1
+ $lib.debug = $cmdopts.debug
2
+
3
+ $lib.print('Testcmd!')
4
+
5
+ if $lib.debug { $lib.print('Debug enabled!') }
@@ -0,0 +1,69 @@
1
+ name: testpkg
2
+ version: 1.2.3
3
+ synapse_version: ">=2.64.0,<3.0.0"
4
+
5
+ genopts:
6
+ dotstorm: true
7
+
8
+ author:
9
+ url: https://vertex.link
10
+ name: The Vertex Project, LLC.
11
+
12
+ desc: The test package does things
13
+
14
+ docs:
15
+ - title: Foo
16
+ path: docs/_build/foo.md
17
+ - title: Bar
18
+ path: docs/_build/bar.md
19
+ - title: Package Documentation
20
+ path: docs/_build/stormpackage.md
21
+
22
+ modules:
23
+ - name: testmod
24
+ - name: apimod
25
+ apidefs:
26
+ - name: search
27
+ desc: |
28
+ Execute a search
29
+
30
+ This API will foo the bar.
31
+
32
+ Examples:
33
+ Foo the ``bar``::
34
+
35
+ yield $lib.import(apimod).search(bar)
36
+
37
+ Baz the bam::
38
+
39
+ yield $lib.import(apimod).search(bam)
40
+
41
+ type:
42
+ type: function
43
+ args:
44
+ - { name: text, type: str, desc: "The text." }
45
+ - { name: mintime, type: [str, int], desc: "The mintime.", default: "-30days" }
46
+ - { name: foo, type: str, desc: "The foo." }
47
+ - { name: bar, type: str, desc: "The bar." }
48
+ - { name: baz, type: str, desc: "The baz." }
49
+ returns:
50
+ name: yields
51
+ type: node
52
+ desc: Yields it:dev:str nodes.
53
+ - name: status
54
+ desc: Get the status of the foo.
55
+ type:
56
+ type: function
57
+ returns:
58
+ type: dict
59
+ desc: A status dictionary.
60
+
61
+ commands:
62
+ - name: testcmd
63
+ descr: |
64
+ A testcmd!
65
+ cmdargs:
66
+ - - --debug
67
+ - default: false
68
+ action: store_true
69
+ help: Show verbose debug output.
@@ -5785,6 +5785,12 @@ class CortexBasicTest(s_t_utils.SynTest):
5785
5785
  ddef = await core01.callStorm('return($lib.dmon.get($iden))', opts=opts)
5786
5786
  self.none(ddef)
5787
5787
 
5788
+ await core00.callStorm('queue.del hehe')
5789
+ await core01.sync()
5790
+
5791
+ self.none(await core00.getAuthGate('queue:hehe'))
5792
+ self.none(await core01.getAuthGate('queue:hehe'))
5793
+
5788
5794
  # now lets start up in the opposite order...
5789
5795
  async with self.getTestCore(dirn=path01, conf=core01conf) as core01:
5790
5796
 
@@ -5889,7 +5895,7 @@ class CortexBasicTest(s_t_utils.SynTest):
5889
5895
 
5890
5896
  # consumer offline
5891
5897
  await asyncio.sleep(0)
5892
- await self.asyncraises(ConnectionRefusedError, core00.callStorm(strim, opts=opts))
5898
+ await self.asyncraises(s_exc.LinkErr, core00.callStorm(strim, opts=opts))
5893
5899
 
5894
5900
  # admin can still cull and break the mirror
5895
5901
  await core00.nodes('[ inet:ipv4=127.0.0.1/28 ]')
@@ -8478,6 +8484,14 @@ class CortexBasicTest(s_t_utils.SynTest):
8478
8484
  self.eq(msgs[1].get('hash'), qhash)
8479
8485
  self.eq(msgs[1].get('pool:from'), f'00.core.{ahanet}')
8480
8486
 
8487
+ with self.getLoggerStream('synapse') as stream:
8488
+ core01.boss.is_shutdown = True
8489
+ self.stormHasNoWarnErr(await core00.stormlist('inet:asn=0'))
8490
+ core01.boss.is_shutdown = False
8491
+
8492
+ stream.seek(0)
8493
+ self.isin('Proxy for pool mirror [01.core.synapse] is shutting down. Skipping.', stream.read())
8494
+
8481
8495
  with patch('synapse.cortex.CoreApi.getNexsIndx', _hang):
8482
8496
 
8483
8497
  with self.getLoggerStream('synapse') as stream:
@@ -8761,3 +8775,8 @@ class CortexBasicTest(s_t_utils.SynTest):
8761
8775
  self.eq(core._test_post_service_storage_index, offs)
8762
8776
  self.eq(core._test_pre_nexus_index, offs)
8763
8777
  self.ge(core._test_post_nexus_index, core._test_pre_nexus_index)
8778
+
8779
+ async def test_cortex_queue_mirror_authgates(self):
8780
+ async with self.getRegrCore('2.213.0-queue-authgates') as core:
8781
+ self.nn(await core.getAuthGate('queue:stillhere'))
8782
+ self.none(await core.getAuthGate('queue:authtest'))
@@ -98,7 +98,7 @@ class DaemonTest(s_t_utils.SynTest):
98
98
 
99
99
  # Invalid data casues a link to fail on rx
100
100
  async with await prox.getPoolLink() as link:
101
- with self.getAsyncLoggerStream('synapse.lib.link', 'rx error') as stream:
101
+ with self.getAsyncLoggerStream('synapse.lib.link', 'rx closed unexpectedly') as stream:
102
102
  byts = b'\x16\x03\x01\x02\x00\x01\x00\x01\xfc\x03\x03\xa6\xa3D\xd5\xdf%\xac\xa9\x92\xc3'
103
103
  await link.send(byts)
104
104
  self.true(await stream.wait(timeout=6))
synapse/tests/test_exc.py CHANGED
@@ -54,3 +54,9 @@ class ExcTest(s_t_utils.SynTest):
54
54
 
55
55
  with self.raises(s_exc.BadArg):
56
56
  s_exc.StormRaise(mesg='newp')
57
+
58
+ async def test_reprexc(self):
59
+ exc = s_exc.SynErr(mesg='woot')
60
+ self.eq('woot', s_exc.reprexc(exc))
61
+ self.eq('ValueError()', s_exc.reprexc(ValueError()))
62
+ self.eq("ValueError('woot')", s_exc.reprexc(ValueError('woot')))
@@ -2192,17 +2192,24 @@ class AstTest(s_test.SynTest):
2192
2192
  self.eq(nodes[1].ndef, ('test:str', 'init2'))
2193
2193
  self.eq(nodes[1].get('hehe'), 'hi')
2194
2194
 
2195
- # Non-runtsafe init fails to execute
2195
+ # Non-runtsafe values allowed in init
2196
+ q = '''
2197
+ init {
2198
+ [test:str=cool :hehe=runtsafety]
2199
+ $lib.print(:hehe)
2200
+ }
2201
+ '''
2202
+ msgs = await core.stormlist(q)
2203
+ self.stormIsInPrint('runtsafety', msgs)
2204
+
2205
+ # Non-runtsafe init doesn't create the node
2196
2206
  q = '''
2197
2207
  test:str^=init +:hehe $hehe=:hehe
2198
2208
  init {
2199
2209
  [test:str=$hehe]
2200
2210
  }
2201
2211
  '''
2202
- msgs = await core.stormlist(q)
2203
- erfo = [m for m in msgs if m[0] == 'err'][0]
2204
- self.eq(erfo[1][0], 'StormRuntimeError')
2205
- self.eq(erfo[1][1].get('mesg'), 'Init block query must be runtsafe')
2212
+ self.len(2, await core.nodes(q))
2206
2213
 
2207
2214
  # Runtsafe init works and can yield nodes, this has inbound nodes as well
2208
2215
  q = '''
@@ -2231,7 +2238,7 @@ class AstTest(s_test.SynTest):
2231
2238
  self.eq(nodes[1].ndef, ('test:str', 'fini2'))
2232
2239
  self.eq(nodes[1].get('hehe'), 'hehe')
2233
2240
 
2234
- # Non-runtsafe fini example which fails
2241
+ # Non-runtsafe fini example
2235
2242
  q = '''
2236
2243
  [test:str=fini3 :hehe="number3"]
2237
2244
  $hehe=:hehe
@@ -2239,10 +2246,10 @@ class AstTest(s_test.SynTest):
2239
2246
  [(test:str=fini4 :hehe=$hehe)]
2240
2247
  }
2241
2248
  '''
2242
- msgs = await core.stormlist(q)
2243
- erfo = [m for m in msgs if m[0] == 'err'][0]
2244
- self.eq(erfo[1][0], 'StormRuntimeError')
2245
- self.eq(erfo[1][1].get('mesg'), 'Fini block query must be runtsafe')
2249
+ nodes = await core.nodes(q)
2250
+ self.len(2, nodes)
2251
+ for node in nodes:
2252
+ self.eq('number3', node.get('hehe'))
2246
2253
 
2247
2254
  # Tally use - case example for counting
2248
2255
  q = '''
@@ -2413,12 +2420,21 @@ class AstTest(s_test.SynTest):
2413
2420
  self.stormIsInPrint('blorp', msgs)
2414
2421
 
2415
2422
  q = '''
2416
- [test:str=latte :hehe=milk] $beep=:hehe | spin | empty { $lib.print($beep) }
2423
+ [test:str=latte :hehe=milk] $beep=:hehe | spin | empty { $lib.print($beep) $lib.print(ok) }
2417
2424
  '''
2418
2425
  msgs = await core.stormlist(q)
2419
2426
  nodes = [m[1] for m in msgs if m[0] == 'node']
2420
2427
  self.len(0, nodes)
2421
- self.stormIsInErr('Empty block query must be runtsafe', msgs)
2428
+ self.stormIsInPrint('ok', msgs)
2429
+ self.stormNotInPrint('milk', msgs)
2430
+
2431
+ q = '''
2432
+ empty { [test:str=runtsafety :hehe=optional] $beep=:hehe $lib.print($beep) }
2433
+ '''
2434
+ msgs = await core.stormlist(q)
2435
+ nodes = [m[1] for m in msgs if m[0] == 'node']
2436
+ self.len(1, nodes)
2437
+ self.stormIsInPrint('optional', msgs)
2422
2438
 
2423
2439
  q = '''
2424
2440
  function foo() {
@@ -3224,11 +3240,11 @@ class AstTest(s_test.SynTest):
3224
3240
  off, end = errm[1][1]['highlight']['offsets']
3225
3241
  self.eq(':foo:bar', text[off:end])
3226
3242
 
3227
- text = 'init { $foo = :bar }'
3243
+ text = 'init { $foo = $bar }'
3228
3244
  msgs = await core.stormlist(text)
3229
3245
  errm = [m for m in msgs if m[0] == 'err'][0]
3230
3246
  off, end = errm[1][1]['highlight']['offsets']
3231
- self.eq(':bar', text[off:end])
3247
+ self.eq('bar', text[off:end])
3232
3248
 
3233
3249
  text = 'inet:ipv5'
3234
3250
  msgs = await core.stormlist(text)
@@ -3554,6 +3570,45 @@ class AstTest(s_test.SynTest):
3554
3570
 
3555
3571
  self.len(1, nodes)
3556
3572
 
3573
+ async def test_ast_subgraph_multipivot(self):
3574
+ async with self.getTestCore() as core:
3575
+ guid = s_common.guid()
3576
+ await core.nodes('''[
3577
+ (test:guid=$guid :size=1234 :tick=now) +(refs)> {[test:str=blorp]}
3578
+ ]''', opts={'vars': {'guid': guid}})
3579
+
3580
+ opts = {
3581
+ 'graph': {
3582
+ 'pivots': ('-> *', '-(refs)> *'),
3583
+ 'refs': True,
3584
+ 'degrees': 1
3585
+ }
3586
+ }
3587
+
3588
+ nodes = []
3589
+ async with await core.snap() as snap:
3590
+ async for node, path in snap.storm('test:guid', opts=opts):
3591
+ nodes.append(node)
3592
+
3593
+ opts = {
3594
+ 'graph': {
3595
+ 'pivots': ('-(refs)> *', '-> *'),
3596
+ 'refs': True,
3597
+ 'degrees': 1
3598
+ }
3599
+ }
3600
+ nodes2 = []
3601
+ async with await core.snap() as snap:
3602
+ async for node, path in snap.storm('test:guid', opts=opts):
3603
+ nodes2.append(node)
3604
+
3605
+ self.eq(set(n.iden() for n in nodes), set(n.iden() for n in nodes2))
3606
+ self.len(3, nodes)
3607
+ ndefs = [n.ndef for n in nodes]
3608
+ self.isin(('test:guid', guid), ndefs)
3609
+ self.isin(('test:str', 'blorp'), ndefs)
3610
+ self.isin(('test:int', 1234), ndefs)
3611
+
3557
3612
  async def test_ast_double_init_fini(self):
3558
3613
  async with self.getTestCore() as core:
3559
3614
  q = '''
@@ -62,3 +62,11 @@ class BossTest(s_test.SynTest):
62
62
  coro = boss.schedCoro(double_promote())
63
63
  self.true(await stream.wait(timeout=6))
64
64
  await coro
65
+
66
+ async with boss.shutdown_lock:
67
+ with self.raises(s_exc.ShuttingDown):
68
+ boss.reqNotShut()
69
+
70
+ boss.is_shutdown = True
71
+ with self.raises(s_exc.ShuttingDown):
72
+ boss.reqNotShut()
@@ -187,6 +187,20 @@ testDataSchema_v1 = {
187
187
  'size': {'type': 'number'},
188
188
  'stuff': {'type': ['number', 'null'], 'default': None},
189
189
  'woot': {'type': 'string'},
190
+ 'blorp': {
191
+ 'type': 'object',
192
+ 'properties': {
193
+ 'bleep': {
194
+ 'type': 'array',
195
+ 'items': {
196
+ 'type': 'object',
197
+ 'properties': {
198
+ 'neato': {'type': 'string'}
199
+ }
200
+ }
201
+ }
202
+ }
203
+ }
190
204
  },
191
205
  'required': ['type', 'size', 'woot'],
192
206
  'additionalProperties': False,
@@ -194,6 +208,18 @@ testDataSchema_v1 = {
194
208
 
195
209
  class CellTest(s_t_utils.SynTest):
196
210
 
211
+ async def test_cell_getLocalUrl(self):
212
+ with self.getTestDir() as dirn:
213
+ async with self.getTestCell(dirn=dirn) as cell:
214
+ url = cell.getLocalUrl()
215
+ self.eq(url, f'cell://root@{dirn}')
216
+
217
+ url = cell.getLocalUrl(share='*/layer')
218
+ self.eq(url, f'cell://root@{dirn}:*/layer')
219
+
220
+ url = cell.getLocalUrl(user='lowuser', share='*/view')
221
+ self.eq(url, f'cell://lowuser@{dirn}:*/view')
222
+
197
223
  async def test_cell_drive(self):
198
224
 
199
225
  with self.getTestDir() as dirn:
@@ -232,13 +258,21 @@ class CellTest(s_t_utils.SynTest):
232
258
  with self.raises(s_exc.BadVersion):
233
259
  await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0)
234
260
 
235
- info = {'name': 'win32k.sys', 'type': 'woot'}
261
+ info = {'name': 'win32k.sys', 'type': 'woot', 'perm': {'users': {}}}
236
262
  info = await cell.addDriveItem(info, reldir=rootdir)
263
+ self.notin('perm', info)
264
+ self.eq(info[0]['permissions'], {
265
+ 'users': {},
266
+ 'roles': {}
267
+ })
237
268
 
238
269
  iden = info[-1].get('iden')
239
270
 
240
271
  tick = s_common.now()
241
272
  rootuser = cell.auth.rootuser.iden
273
+ fooser = await cell.auth.addUser('foo')
274
+ neatrole = await cell.auth.addRole('neatrole')
275
+ await fooser.grant(neatrole.iden)
242
276
 
243
277
  with self.raises(s_exc.SchemaViolation):
244
278
  versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser}
@@ -293,6 +327,10 @@ class CellTest(s_t_utils.SynTest):
293
327
  info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 17, 'stuff': 15})
294
328
  self.eq(versinfo, (await cell.getDriveData(iden))[0])
295
329
 
330
+ await cell.setDriveItemProp(iden, versinfo, ('stuff',), 1234)
331
+ data = await cell.getDriveData(iden)
332
+ self.eq(data[1]['stuff'], 1234)
333
+
296
334
  # This will be done by the cell in a cell storage version migration...
297
335
  async def migrate_v1(info, versinfo, data):
298
336
  data['woot'] = 'woot'
@@ -300,6 +338,37 @@ class CellTest(s_t_utils.SynTest):
300
338
 
301
339
  await cell.drive.setTypeSchema('woot', testDataSchema_v1, migrate_v1)
302
340
 
341
+ versinfo['version'] = (1, 1, 1)
342
+ await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829)
343
+ data = await cell.getDriveData(iden)
344
+ self.eq(data[0]['version'], (1, 1, 1))
345
+ self.eq(data[1]['stuff'], 3829)
346
+
347
+ await self.asyncraises(s_exc.NoSuchIden, cell.setDriveItemProp(s_common.guid(), versinfo, ('lolnope',), 'not real'))
348
+
349
+ await self.asyncraises(s_exc.BadArg, cell.setDriveItemProp(iden, versinfo, ('blorp', 0, 'neato'), 'my special string'))
350
+ data[1]['blorp'] = {
351
+ 'bleep': [{'neato': 'thing'}]
352
+ }
353
+ info, versinfo = await cell.setDriveData(iden, versinfo, data[1])
354
+ now = s_common.now()
355
+ versinfo['updated'] = now
356
+ await cell.setDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato'), 'my special string')
357
+ data = await cell.getDriveData(iden)
358
+ self.eq(now, data[0]['updated'])
359
+ self.eq('my special string', data[1]['blorp']['bleep'][0]['neato'])
360
+
361
+ versinfo['version'] = (1, 2, 1)
362
+ await cell.delDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato'))
363
+ vers, data = await cell.getDriveData(iden)
364
+ self.eq((1, 2, 1), vers['version'])
365
+ self.nn(data['blorp']['bleep'][0])
366
+ self.notin('neato', data['blorp']['bleep'][0])
367
+
368
+ await self.asyncraises(s_exc.NoSuchIden, cell.delDriveItemProp(s_common.guid(), versinfo, 'blorp'))
369
+
370
+ self.none(await cell.delDriveItemProp(iden, versinfo, ('lolnope', 'nopath')))
371
+
303
372
  versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0))
304
373
  self.eq('woot', data.get('woot'))
305
374
 
@@ -313,10 +382,10 @@ class CellTest(s_t_utils.SynTest):
313
382
  await cell.getDriveInfo(iden, typename='newp')
314
383
 
315
384
  self.nn(await cell.getDriveInfo(iden))
316
- self.len(2, [vers async for vers in cell.getDriveDataVersions(iden)])
385
+ self.len(4, [vers async for vers in cell.getDriveDataVersions(iden)])
317
386
 
318
387
  await cell.delDriveData(iden)
319
- self.len(1, [vers async for vers in cell.getDriveDataVersions(iden)])
388
+ self.len(3, [vers async for vers in cell.getDriveDataVersions(iden)])
320
389
 
321
390
  await cell.delDriveInfo(iden)
322
391
 
@@ -340,8 +409,12 @@ class CellTest(s_t_utils.SynTest):
340
409
  baziden = pathinfo[2].get('iden')
341
410
  self.eq(pathinfo, await cell.drive.getItemPath(baziden))
342
411
 
343
- info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: 3}, 'roles': {}})
344
- self.eq(3, info['perm']['users'][rootuser])
412
+ info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: s_cell.PERM_ADMIN}, 'roles': {}})
413
+ # make sure drive perms work with easy perms
414
+ self.true(cell._hasEasyPerm(info, cell.auth.rootuser, s_cell.PERM_ADMIN))
415
+ # defaults to READ
416
+ self.true(cell._hasEasyPerm(info, fooser, s_cell.PERM_READ))
417
+ self.false(cell._hasEasyPerm(info, fooser, s_cell.PERM_EDIT))
345
418
 
346
419
  with self.raises(s_exc.NoSuchIden):
347
420
  # s_drive.rootdir is all 00s... ;)
@@ -636,6 +709,44 @@ class CellTest(s_t_utils.SynTest):
636
709
  with self.raises(s_exc.NeedConfValu):
637
710
  await echo.reqAhaProxy()
638
711
 
712
+ async def test_cell_drive_perm_migration(self):
713
+ async with self.getRegrCore('drive-perm-migr') as core:
714
+ item = await core.getDrivePath('driveitemdefaultperms')
715
+ self.len(1, item)
716
+ self.notin('perm', item)
717
+ self.eq(item[0]['permissions'], {'users': {}, 'roles': {}})
718
+
719
+ ldog = await core.auth.getRoleByName('littledog')
720
+ bdog = await core.auth.getRoleByName('bigdog')
721
+
722
+ louis = await core.auth.getUserByName('lewis')
723
+ tim = await core.auth.getUserByName('tim')
724
+ mj = await core.auth.getUserByName('mj')
725
+
726
+ item = await core.getDrivePath('permfolder/driveitemwithperms')
727
+ self.len(2, item)
728
+ self.notin('perm', item[0])
729
+ self.notin('perm', item[1])
730
+ self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}})
731
+ self.eq(item[1]['permissions'], {
732
+ 'users': {
733
+ mj.iden: s_cell.PERM_ADMIN
734
+ },
735
+ 'roles': {
736
+ ldog.iden: s_cell.PERM_READ,
737
+ bdog.iden: s_cell.PERM_EDIT,
738
+ },
739
+ 'default': s_cell.PERM_DENY
740
+ })
741
+
742
+ # make sure it's all good with easy perms
743
+ self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN))
744
+ self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT))
745
+
746
+ self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN))
747
+ self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ))
748
+ self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT))
749
+
639
750
  async def test_cell_unix_sock(self):
640
751
 
641
752
  async with self.getTestCore() as core:
@@ -2580,10 +2691,10 @@ class CellTest(s_t_utils.SynTest):
2580
2691
  viewiden = view.get('iden')
2581
2692
 
2582
2693
  opts = {'view': viewiden}
2583
- with self.getLoggerStream('synapse.lib.lmdbslab',
2694
+ with self.getAsyncLoggerStream('synapse.lib.lmdbslab',
2584
2695
  'Error during slab resize callback - foo') as stream:
2585
- nodes = await core.stormlist('for $x in $lib.range(200) {[inet:ipv4=$x]}', opts=opts)
2586
- self.true(stream.wait(1))
2696
+ msgs = await core.stormlist('for $x in $lib.range(200) {[test:int=$x]}', opts=opts)
2697
+ self.true(await stream.wait(timeout=30))
2587
2698
 
2588
2699
  async with self.getTestCore() as core:
2589
2700