QuLab 2.3.3__cp311-cp311-win_amd64.whl → 2.3.5__cp311-cp311-win_amd64.whl

{QuLab-2.3.3.dist-info → QuLab-2.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.3.3
+Version: 2.3.5
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>
@@ -38,6 +38,9 @@ Requires-Dist: ply >=3.11
 Requires-Dist: pyzmq >=25.1.0
 Requires-Dist: scipy >=1.0.0
 Requires-Dist: watchdog >=4.0.0
+Provides-Extra: full
+Requires-Dist: SQLAlchemy >=2.0.19 ; extra == 'full'
+Requires-Dist: uvloop >=0.19.0 ; extra == 'full'
 
 # QuLab
 [![View build status](https://travis-ci.org/feihoo87/QuLab.svg?branch=master)](https://travis-ci.org/feihoo87/QuLab)

{QuLab-2.3.3.dist-info → QuLab-2.3.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=vkFybY8YSsQilYdThPRD83-btPAR41sy_WCXiM-6mME,141
 qulab/__main__.py,sha256=V7iokU7awstgjCeiF_hoOdFyrqJwC_4QetiLe7cWvOQ,454
-qulab/fun.cp311-win_amd64.pyd,sha256=Tum8uq_Huf6Ix76rz3ni4O63xMAIC6TtwYo9pwYhnGA,31744
-qulab/version.py,sha256=_TojZ2sN-5aiITSzim-_br5UuW9pwpo2Tx8vnDWpsqE,21
+qulab/fun.cp311-win_amd64.pyd,sha256=Hdu7_RHp4pqeVgzgSi9vZtC9Utq_N2pFv8thjeReiCk,31744
+qulab/version.py,sha256=ctNy09QkNipfMu2fxyFBsAXtRoAHMP1bEKHsc3zRmkk,21
 qulab/monitor/__init__.py,sha256=xEVDkJF8issrsDeLqQmDsvtRmrf-UiViFcGTWuzdlFU,43
 qulab/monitor/__main__.py,sha256=k2H1H5Zf9LLXTDLISJkbikLH-z0f1e5i5i6wXXYPOrE,105
 qulab/monitor/config.py,sha256=y_5StMkdrbZO1ziyKBrvIkB7Jclp9RCPK1QbsOhCxnY,785
@@ -16,13 +16,13 @@ qulab/scan/__init__.py,sha256=RR_0NQcr8Mi3vpWdypydbijQ1rXA0D3DEidQ7xjNslM,133
 qulab/scan/curd.py,sha256=yaTglGiS6mlk0GqDHi_w8T02XGBMvDZtXSdML7zDywk,7117
 qulab/scan/expression.py,sha256=l7TYBmcJIo0M5GJm1TtrkrMFj5mCmrfLOMCILKbOivU,20712
 qulab/scan/models.py,sha256=ZvXkJEt5Yz3Sjx0JKzYka-q2Uo-w_iVzHgH8A6DbjF0,18236
-qulab/scan/optimize.py,sha256=ACfGSfrFpPgmZo_P5kD4mquahXImYipdP7E86dtFQO8,2635
+qulab/scan/optimize.py,sha256=zOR4Wp96bLarTSiPJ-cTAfT-V_MU-YEgB-XqYsBhS30,2637
 qulab/scan/query.py,sha256=RM8bG4Tcx_PaNk8tv9HdlTZ1dGuuSr3sZVkYVq2BtfQ,12183
 qulab/scan/record.py,sha256=MVmxhIzwmOju7eWxJEWsqJZlVgrDeRXGMfNvXImj7Ms,21883
-qulab/scan/scan.py,sha256=HMeVxoaEWjOj0eBusrQ-QQUFDAc_ivoLv95XkDmOS70,39947
-qulab/scan/server.py,sha256=em2tM_JPySnUNhusvl92ZPUGxz1mM34jFo5v2ji4uy0,19611
-qulab/scan/space.py,sha256=S-jHaXXf12FzjajOhPsZ-mkpGzAxrFAI3tRuQrvTBKg,6274
-qulab/scan/utils.py,sha256=30qnYvyFyctwcWxOCUpCNxXgGysA7xdIDzYbjwxGUzA,3744
+qulab/scan/scan.py,sha256=p7LZvmjDSnUHeszgILlEXFIspHVPLk9Rt2dVqFtNsaQ,40645
+qulab/scan/server.py,sha256=iT9wMEUQ2Pz_5nioHA7nII1MBPpQi-rCFbEJnRdWkDg,20487
+qulab/scan/space.py,sha256=t8caa_gKlnhaAIEksJyxINUTecOS7lMWAz1HDKlVcds,6909
+qulab/scan/utils.py,sha256=YXFA19HEakHB5TaPOU9CjQ1Lc3GUJaSIGOtiWjAWsG4,6353
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=6-EjN0waX1yfcMPJXqpIr9UlrIEsSCFApm5G-ZeaPMQ,1742
 qulab/storage/base_dataset.py,sha256=28y3-OZrqJ52p5sbirEpUgjb7hqwLLpd38KU9DCkD24,12217
@@ -78,9 +78,9 @@ qulab/visualization/plot_layout.py,sha256=yAnMONOms7_szCdng-8wPpUMPis5UnbaNNzV4K
 qulab/visualization/plot_seq.py,sha256=h9D0Yl_yO64IwlvBgzMu9EBKr9gg6y8QE55gu2PfTns,2783
 qulab/visualization/qdat.py,sha256=HubXFu4nfcA7iUzghJGle1C86G6221hicLR0b-GqhKQ,5887
 qulab/visualization/widgets.py,sha256=HcYwdhDtLreJiYaZuN3LfofjJmZcLwjMfP5aasebgDo,3266
-QuLab-2.3.3.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
-QuLab-2.3.3.dist-info/METADATA,sha256=jKqntTp7yF3mnpI4ItBA9tLV0zjlI-v_uRTrf_AonRk,3609
-QuLab-2.3.3.dist-info/WHEEL,sha256=rt4ogJhfwzOIZpQG1-igUOP-t_yXz3fi7xFfo-fNzUo,101
-QuLab-2.3.3.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.3.3.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.3.3.dist-info/RECORD,,
+QuLab-2.3.5.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
+QuLab-2.3.5.dist-info/METADATA,sha256=CUbsOaCfpAwZ7tf7MIywKpb0DuidIos4u6sMgE2PXKs,3735
+QuLab-2.3.5.dist-info/WHEEL,sha256=YqgvAYDJ-im_5PHkg0IY61P_j_tf16xq16unjojo7Hs,101
+QuLab-2.3.5.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.3.5.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.3.5.dist-info/RECORD,,

{QuLab-2.3.3.dist-info → QuLab-2.3.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: setuptools (71.0.0)
 Root-Is-Purelib: false
 Tag: cp311-cp311-win_amd64
 

qulab/scan/optimize.py

@@ -72,5 +72,5 @@ class NgOptimizer():
         if history:
             ret.x_iters = self._all_x
             ret.func_vals = self._all_y
-        ret.fun = recommendation.value
+        # ret.fun = recommendation.value
         return ret

qulab/scan/scan.py

@@ -6,9 +6,7 @@ import itertools
 import lzma
 import os
 import pickle
-import platform
 import re
-import subprocess
 import sys
 import uuid
 from concurrent.futures import ProcessPoolExecutor
@@ -26,7 +24,8 @@ from .optimize import NgOptimizer
 from .record import Record
 from .server import default_record_port
 from .space import Optimizer, OptimizeSpace, Space
-from .utils import async_zip, call_function, dump_globals
+from .utils import (async_zip, call_function, dump_dict, dump_globals,
+                    get_installed_packages, get_system_info, yapf_reformat)
 
 try:
     from tqdm.notebook import tqdm
@@ -58,57 +57,34 @@ else:
     default_executor = default_server
 
 
-def yapf_reformat(cell_text):
-    try:
-        import isort
-        import yapf.yapflib.yapf_api
-
-        fname = f"f{uuid.uuid1().hex}"
-
-        def wrap(source):
-            lines = [f"async def {fname}():"]
-            for line in source.split('\n'):
-                lines.append("    " + line)
-            return '\n'.join(lines)
+class Promise():
+    __slots__ = ['task', 'key', 'attr']
 
-        def unwrap(source):
-            lines = []
-            for line in source.split('\n'):
-                if line.startswith(f"async def {fname}():"):
-                    continue
-                lines.append(line[4:])
-            return '\n'.join(lines)
+    def __init__(self, task, key=None, attr=None):
+        self.task = task
+        self.key = key
+        self.attr = attr
 
-        cell_text = re.sub('^%', '#%#', cell_text, flags=re.M)
-        reformated_text = unwrap(
-            yapf.yapflib.yapf_api.FormatCode(wrap(isort.code(cell_text)))[0])
-        return re.sub('^#%#', '%', reformated_text, flags=re.M)
-    except:
-        return cell_text
+    def __await__(self):
 
+        async def _getitem(task, key):
+            return (await task)[key]
 
-def get_installed_packages():
-    result = subprocess.run([sys.executable, '-m', 'pip', 'freeze'],
-                            stdout=subprocess.PIPE,
-                            text=True)
+        async def _getattr(task, attr):
+            return getattr(await task, attr)
 
-    lines = result.stdout.split('\n')
-    packages = []
-    for line in lines:
-        if line:
-            packages.append(line)
-    return packages
+        if self.key is not None:
+            return _getitem(self.task, self.key).__await__()
+        elif self.attr is not None:
+            return _getattr(self.task, self.attr).__await__()
+        else:
+            return self.task.__await__()
 
+    def __getitem__(self, key):
+        return Promise(self.task, key, None)
 
-def get_system_info():
-    info = {
-        'OS': platform.uname()._asdict(),
-        'Python': sys.version,
-        'PythonExecutable': sys.executable,
-        'PythonPath': sys.path,
-        'packages': get_installed_packages()
-    }
-    return info
+    def __getattr__(self, attr):
+        return Promise(self.task, None, attr)
 
 
 def current_notebook():
@@ -170,52 +146,185 @@ def _get_depends(func: Callable):
     return args
 
 
-class Promise():
-    __slots__ = ['task', 'key', 'attr']
+def _run_function_in_process(buf):
+    func, args, kwds = dill.loads(buf)
+    return func(*args, **kwds)
 
-    def __init__(self, task, key=None, attr=None):
-        self.task = task
-        self.key = key
-        self.attr = attr
 
-    def __await__(self):
+async def update_variables(variables: dict[str, Any], updates: dict[str, Any],
+                           setters: dict[str, Callable]):
+    coros = []
+    for name, value in updates.items():
+        if name in setters:
+            coro = setters[name](value)
+            if inspect.isawaitable(coro):
+                coros.append(coro)
+        variables[name] = value
+    if coros:
+        await asyncio.gather(*coros)
 
-        async def _getitem(task, key):
-            return (await task)[key]
 
-        async def _getattr(task, attr):
-            return getattr(await task, attr)
+async def _iter_level(variables,
+                      iters: list[tuple[str, Iterable | Expression | Callable
+                                        | OptimizeSpace]],
+                      order: list[list[str]],
+                      functions: dict[str, Callable | Expression],
+                      optimizers: dict[str, Optimizer],
+                      setters: dict[str, Callable] = {},
+                      getters: dict[str, Callable] = {}):
+    iters_d = {}
+    env = Env()
+    env.variables = variables
+    opts = {}
 
-        if self.key is not None:
-            return _getitem(self.task, self.key).__await__()
-        elif self.attr is not None:
-            return _getattr(self.task, self.attr).__await__()
+    for name, iter in iters:
+        if isinstance(iter, OptimizeSpace):
+            if iter.optimizer.name not in opts:
+                opts[iter.optimizer.name] = iter.optimizer.create()
+        elif isinstance(iter, Expression):
+            iters_d[name] = iter.eval(env)
+        elif isinstance(iter, Space):
+            iters_d[name] = iter.toarray()
+        elif callable(iter):
+            iters_d[name] = await call_function(iter, variables)
         else:
-            return self.task.__await__()
+            iters_d[name] = iter
 
-    def __getitem__(self, key):
-        return Promise(self.task, key, None)
+    maxiter = 0xffffffff
+    for name, opt in opts.items():
+        opt_cfg = optimizers[name]
+        maxiter = min(maxiter, opt_cfg.maxiter)
 
-    def __getattr__(self, attr):
-        return Promise(self.task, None, attr)
+    async for args in async_zip(*iters_d.values(), range(maxiter)):
+        await update_variables(variables, dict(zip(iters_d.keys(), args[:-1])),
+                               setters)
+        for name, opt in opts.items():
+            args = opt.ask()
+            opt_cfg = optimizers[name]
+            await update_variables(variables, {
+                n: v
+                for n, v in zip(opt_cfg.dimensions.keys(), args)
+            }, setters)
 
+        await update_variables(
+            variables, await call_many_functions(order, functions, variables),
+            setters)
 
-def _run_function_in_process(buf):
-    func, args, kwds = dill.loads(buf)
-    return func(*args, **kwds)
+        yield variables
 
+        variables.update(await call_many_functions(order, getters, variables))
 
-def _dump_description(description):
-    d = {}
-    for key, value in description.items():
-        if key in [
-                'intrinsic_loops', 'app', 'tags', 'loops',
-                'independent_variables', 'axis', 'config', 'entry'
-        ]:
-            d[key] = value
+        if opts:
+            for key in list(variables.keys()):
+                if key.startswith('*') or ',' in key:
+                    await _unpack(key, variables)
+
+        for name, opt in opts.items():
+            opt_cfg = optimizers[name]
+            args = [variables[n] for n in opt_cfg.dimensions.keys()]
+
+            if name not in variables:
+                raise ValueError(f'{name} not in variables.')
+            fun = variables[name]
+            if inspect.isawaitable(fun):
+                fun = await fun
+            if opt_cfg.minimize:
+                opt.tell(args, fun)
+            else:
+                opt.tell(args, -fun)
+
+    if opts:
+        for name, opt in opts.items():
+            opt_cfg = optimizers[name]
+            result = opt.get_result()
+            await update_variables(
+                variables, {
+                    name: value
+                    for name, value in zip(opt_cfg.dimensions.keys(), result.x)
+                }, setters)
+
+        yield variables
+
+        variables.update(await call_many_functions(order, getters, variables))
+
+        for key in list(variables.keys()):
+            if key.startswith('*') or ',' in key:
+                await _unpack(key, variables)
+
+
+async def call_many_functions(order: list[list[str]],
+                              functions: dict[str, Callable],
+                              variables: dict[str, Any]) -> dict[str, Any]:
+    ret = {}
+    for group in order:
+        waited = []
+        coros = []
+        for name in group:
+            if name in functions:
+                waited.append(name)
+                coros.append(call_function(functions[name], variables | ret))
+        if coros:
+            results = await asyncio.gather(*coros)
+            ret.update(dict(zip(waited, results)))
+    return ret
+
+
+async def _unpack(key, variables):
+    x = variables[key]
+    if inspect.isawaitable(x):
+        x = await x
+    if key.startswith('**'):
+        assert isinstance(
+            x, dict), f"Should promise a dict for `**` symbol. {key}"
+        if "{key}" in key:
+            for k, v in x.items():
+                variables[key[2:].format(key=k)] = v
+        else:
+            variables.update(x)
+    elif key.startswith('*'):
+        assert isinstance(
+            x, (list, tuple,
+                np.ndarray)), f"Should promise a list for `*` symbol. {key}"
+        for i, v in enumerate(x):
+            k = key[1:].format(i=i)
+            variables[k] = v
+    elif ',' in key:
+        keys1, keys2 = [], []
+        args = None
+        for k in key.split(','):
+            if k.startswith('*'):
+                if args is None:
+                    args = k
+                else:
+                    raise ValueError(f'Only one `*` symbol is allowed. {key}')
+            elif args is None:
+                keys1.append(k)
+            else:
+                keys2.append(k)
+        assert isinstance(
+            x,
+            (list, tuple,
+             np.ndarray)), f"Should promise a list for multiple symbols. {key}"
+        if args is None:
+            assert len(keys1) == len(
+                x), f"Length of keys and values should be equal. {key}"
+            for k, v in zip(keys1, x):
+                variables[k] = v
         else:
-            d[key] = dill.dumps(value)
-    return dill.dumps(description)
+            assert len(keys1) + len(keys2) <= len(
+                x), f"Too many values for unpacking. {key}"
+            for k, v in zip(keys1, x[:len(keys1)]):
+                variables[k] = v
+            end = -len(keys2) if keys2 else None
+            for i, v in enumerate(x[len(keys1):end]):
+                k = args[1:].format(i=i)
+                variables[k] = v
+            if keys2:
+                for k, v in zip(keys2, x[end:]):
+                    variables[k] = v
+    else:
+        return
+    del variables[key]
 
 
 class Scan():
@@ -245,6 +354,7 @@ class Scan():
         self.id = task_uuid()
         self.record = None
         self.config = {} if config is None else copy.deepcopy(config)
+        self._raw_config_copy = copy.deepcopy(self.config)
         self.description = {
             'app': app,
             'tags': tags,
@@ -265,7 +375,7 @@ class Scan():
             'filters': {},
             'total': {},
             'database': database,
-            'hiden': ['self', 'config', r'^__.*', r'.*__$'],
+            'hiden': ['self', 'config', r'^__.*', r'.*__$', r'^#.*'],
             'entry': {
                 'system': get_system_info(),
                 'env': {},
@@ -277,6 +387,7 @@ class Scan():
         self._current_level = 0
         self._variables = {}
         self._main_task = None
+        self._background_tasks = ()
         self._sock = None
         self._sem = asyncio.Semaphore(max_promise + 1)
         self._bar: dict[int, tqdm] = {}
@@ -300,6 +411,7 @@ class Scan():
         del state['record']
         del state['_sock']
         del state['_main_task']
+        del state['_background_tasks']
         del state['_bar']
         del state['_msg_queue']
         del state['_prm_queue']
@@ -312,6 +424,7 @@ class Scan():
         self.record = None
         self._sock = None
         self._main_task = None
+        self._background_tasks = ()
         self._bar = {}
         self._prm_queue = asyncio.Queue()
         self._msg_queue = asyncio.Queue(self._max_message)
@@ -396,7 +509,7 @@ class Scan():
 
         if self.config:
             self.description['config'] = await create_config(
-                self.config, self.description['database'], self._sock)
+                self._raw_config_copy, self.description['database'], self._sock)
         if current_notebook() is None:
             await create_notebook('untitle', self.description['database'],
                                   self._sock)
@@ -412,7 +525,11 @@ class Scan():
             'method':
             'record_create',
             'description':
-            _dump_description(self.description)
+            dump_dict(self.description,
+                      keys=[
+                          'intrinsic_loops', 'app', 'tags', 'loops',
+                          'independent_variables', 'axis', 'config', 'entry'
+                      ])
         })
 
         record_id = await self._sock.recv_pyobj()
@@ -470,9 +587,9 @@ class Scan():
             if depends:
                 self.add_depends(name, depends)
                 s = ','.join(depends)
-                self.description['functions'][f'_tmp_{name}'] = value
+                self.description['functions'][f'#{name}'] = value
                 self.description['functions'][name] = eval(
-                    f"lambda self, {s}: self.description['functions']['_tmp_{name}']({s})"
+                    f"lambda self, {s}: self.description['functions']['#{name}']({s})"
                 )
             else:
                 self.add_depends(name, _get_depends(value))
@@ -612,27 +729,33 @@ class Scan():
             await task
             self._msg_queue.task_done()
 
+    @contextlib.asynccontextmanager
+    async def _send_msg_and_update_bar(self):
+        send_msg_task = asyncio.create_task(self._send_msg())
+        update_progress_task = asyncio.create_task(self._update_progress())
+        try:
+            yield (send_msg_task, update_progress_task)
+        finally:
+            update_progress_task.cancel()
+            send_msg_task.cancel()
+            while True:
+                try:
+                    task = self._prm_queue.get_nowait()
+                except:
+                    break
+                try:
+                    task.cancel()
+                except:
+                    pass
+
+    async def _check_background_tasks(self):
+        for task in self._background_tasks:
+            if task.done():
+                await task
+
     async def run(self):
         assymbly(self.description)
-
-        @contextlib.asynccontextmanager
-        async def send_msg_and_update_bar(self):
-            send_msg_task = asyncio.create_task(self._send_msg())
-            update_progress_task = asyncio.create_task(self._update_progress())
-            try:
-                yield
-            finally:
-                update_progress_task.cancel()
-                send_msg_task.cancel()
-                while True:
-                    try:
-                        task = self._prm_queue.get_nowait()
-                    except:
-                        break
-                    try:
-                        task.cancel()
-                    except:
-                        pass
+        self._background_tasks = ()
 
         if isinstance(
                 self.description['database'],
@@ -641,12 +764,14 @@ class Scan():
                                          connect=self.description['database'],
                                          socket=self._sock) as socket:
                 self._sock = socket
-                async with send_msg_and_update_bar(self):
+                async with self._send_msg_and_update_bar() as background_tasks:
+                    self._background_tasks = background_tasks
                     await self._run()
         else:
             if self.config:
-                self.description['config'] = copy.deepcopy(self.config)
-            async with send_msg_and_update_bar(self):
+                self.description['config'] = self._raw_config_copy
+            async with self._send_msg_and_update_bar() as background_tasks:
+                self._background_tasks = background_tasks
                 await self._run()
 
     async def _run(self):
@@ -671,11 +796,11 @@ class Scan():
             self.description['functions'], self.variables)
         await update_variables(self.variables, updates,
                                self.description['setters'])
-
+        await self._check_background_tasks()
         await self.work()
         for level, bar in self._bar.items():
             bar.close()
-
+        await self._check_background_tasks()
         if self._single_step:
             self.variables.update(await call_many_functions(
                 self.description['order'].get(-1, []),
@@ -683,7 +808,7 @@ class Scan():
 
             await self.emit(0, 0, 0, self.variables)
             await self.emit(-1, 0, 0, {})
-
+        await self._check_background_tasks()
         await self._prm_queue.join()
         await self._msg_queue.join()
         return self.variables
@@ -747,6 +872,7 @@ class Scan():
                 | {'config': self._synchronize_config},
                 self.description['optimizers'], self.description['setters'],
                 self.description['getters']):
+            await self._check_background_tasks()
             self._current_level += 1
             if await self._filter(variables, self.current_level - 1):
                 yield variables
@@ -758,11 +884,13 @@ class Scan():
             self._current_level -= 1
             self._prm_queue.put_nowait(
                 self._update_progress_bar(self.current_level, 1))
+            await self._check_background_tasks()
         if self.current_level == 0:
             await self.emit(self.current_level - 1, 0, 0, {})
             for name, value in self.variables.items():
                 if inspect.isawaitable(value):
                     self.variables[name] = await value
+            await self._check_background_tasks()
             await self._prm_queue.join()
 
     async def work(self, **kwds):
@@ -823,7 +951,7 @@ class Scan():
             return await awaitable
 
 
-def assymbly(description):
+def _get_environment(description):
     import __main__
     from IPython import get_ipython
 
@@ -853,6 +981,10 @@ def assymbly(description):
 
     description['entry']['env'] = {k: v for k, v in os.environ.items()}
 
+    return description
+
+
+def _mapping_levels(description):
     mapping = {
         label: level
         for level, label in enumerate(
@@ -885,8 +1017,23 @@ def assymbly(description):
                                                   len(space))
             except:
                 pass
+    return levels
+
 
+def _get_independent_variables(description):
+    independent_variables = set(description['intrinsic_loops'].keys())
+    for level, loops in description['loops'].items():
+        for name, iterable in loops:
+            if isinstance(iterable, (np.ndarray, list, tuple, range, Space)):
+                independent_variables.add(name)
+    return independent_variables
+
+
+def _build_dependents(description, levels, independent_variables):
     dependents = copy.deepcopy(description['dependents'])
+    all_nodes = set(description['dependents'].keys())
+    for key, deps in dependents.items():
+        all_nodes.update(deps)
 
     for level in levels:
         range_list = description['loops'].get(level, [])
@@ -899,6 +1046,13 @@ def assymbly(description):
                 dependents[name] = set()
             dependents[name].add(f'#__loop_{level}')
 
+    after_yield = set()
+    for key in all_nodes:
+        if key not in independent_variables and key not in description[
+                'consts']:
+            if key not in dependents:
+                after_yield.add(key)
+
     def _get_all_depends(key, graph):
         ret = set()
         if key not in graph:
@@ -912,7 +1066,13 @@ def assymbly(description):
     full_depends = {}
     for key in dependents:
         full_depends[key] = _get_all_depends(key, dependents)
+        if full_depends[key] & after_yield:
+            after_yield.add(key)
+
+    return dependents, full_depends, after_yield
+
 
+def _build_order(description, levels, dependents, full_depends):
     levels = {}
     passed = set()
     all_keys = set(description['consts'].keys())
@@ -957,8 +1117,9 @@ def assymbly(description):
                 description['order'][level].append(ready)
                 keys -= set(ready)
 
+
+def _make_axis(description):
     axis = {}
-    independent_variables = set(description['intrinsic_loops'].keys())
 
     for name in description['consts']:
         axis[name] = ()
@@ -967,8 +1128,6 @@ def assymbly(description):
             if isinstance(iterable, OptimizeSpace):
                 axis[name] = tuple(range(level + 1))
                 continue
-            elif isinstance(iterable, (np.ndarray, list, tuple, range, Space)):
-                independent_variables.add(name)
             axis[name] = (level, )
 
     for level, group in description['order'].items():
@@ -989,176 +1148,19 @@ def assymbly(description):
         k: tuple([x for x in v if x >= 0])
         for k, v in axis.items()
     }
-    description['independent_variables'] = independent_variables
-
-    return description
-
-
-async def update_variables(variables: dict[str, Any], updates: dict[str, Any],
-                           setters: dict[str, Callable]):
-    coros = []
-    for name, value in updates.items():
-        if name in setters:
-            coro = setters[name](value)
-            if inspect.isawaitable(coro):
-                coros.append(coro)
-        variables[name] = value
-    if coros:
-        await asyncio.gather(*coros)
-
-
-async def _iter_level(variables,
-                      iters: list[tuple[str, Iterable | Expression | Callable
-                                        | OptimizeSpace]],
-                      order: list[list[str]],
-                      functions: dict[str, Callable | Expression],
-                      optimizers: dict[str, Optimizer],
-                      setters: dict[str, Callable] = {},
-                      getters: dict[str, Callable] = {}):
-    iters_d = {}
-    env = Env()
-    env.variables = variables
-    opts = {}
 
-    for name, iter in iters:
-        if isinstance(iter, OptimizeSpace):
-            if iter.optimizer.name not in opts:
-                opts[iter.optimizer.name] = iter.optimizer.create()
-        elif isinstance(iter, Expression):
-            iters_d[name] = iter.eval(env)
-        elif isinstance(iter, Space):
-            iters_d[name] = iter.toarray()
-        elif callable(iter):
-            iters_d[name] = await call_function(iter, variables)
-        else:
-            iters_d[name] = iter
-
-    maxiter = 0xffffffff
-    for name, opt in opts.items():
-        opt_cfg = optimizers[name]
-        maxiter = min(maxiter, opt_cfg.maxiter)
-
-    async for args in async_zip(*iters_d.values(), range(maxiter)):
-        await update_variables(variables, dict(zip(iters_d.keys(), args[:-1])),
-                               setters)
-        for name, opt in opts.items():
-            args = opt.ask()
-            opt_cfg = optimizers[name]
-            await update_variables(variables, {
-                n: v
-                for n, v in zip(opt_cfg.dimensions.keys(), args)
-            }, setters)
-
-        await update_variables(
-            variables, await call_many_functions(order, functions, variables),
-            setters)
-
-        yield variables
-
-        variables.update(await call_many_functions(order, getters, variables))
-
-        if opts:
-            for key in list(variables.keys()):
-                if key.startswith('*') or ',' in key:
-                    await _unpack(key, variables)
-
-        for name, opt in opts.items():
-            opt_cfg = optimizers[name]
-            args = [variables[n] for n in opt_cfg.dimensions.keys()]
-
-            if name not in variables:
-                raise ValueError(f'{name} not in variables.')
-            fun = variables[name]
-            if inspect.isawaitable(fun):
-                fun = await fun
-            if opt_cfg.minimize:
-                opt.tell(args, fun)
-            else:
-                opt.tell(args, -fun)
-
-    for name, opt in opts.items():
-        opt_cfg = optimizers[name]
-        result = opt.get_result()
-        await update_variables(
-            variables, {
-                name: value
-                for name, value in zip(opt_cfg.dimensions.keys(), result.x)
-            }, setters)
-        variables[name] = result.fun
-    if opts:
-        yield variables
 
+def assymbly(description):
+    _get_environment(description)
+    levels = _mapping_levels(description)
+    independent_variables = _get_independent_variables(description)
+    description['independent_variables'] = independent_variables
 
-async def call_many_functions(order: list[list[str]],
-                              functions: dict[str, Callable],
-                              variables: dict[str, Any]) -> dict[str, Any]:
-    ret = {}
-    for group in order:
-        waited = []
-        coros = []
-        for name in group:
-            if name in functions:
-                waited.append(name)
-                coros.append(call_function(functions[name], variables | ret))
-        if coros:
-            results = await asyncio.gather(*coros)
-            ret.update(dict(zip(waited, results)))
-    return ret
+    dependents, full_depends, after_yield = _build_dependents(
+        description, levels, independent_variables)
 
+    _build_order(description, levels, dependents, full_depends)
+    _make_axis(description)
 
-async def _unpack(key, variables):
-    x = variables[key]
-    if inspect.isawaitable(x):
-        x = await x
-    if key.startswith('**'):
-        assert isinstance(
-            x, dict), f"Should promise a dict for `**` symbol. {key}"
-        if "{key}" in key:
-            for k, v in x.items():
-                variables[key[2:].format(key=k)] = v
-        else:
-            variables.update(x)
-    elif key.startswith('*'):
-        assert isinstance(
-            x, (list, tuple,
-                np.ndarray)), f"Should promise a list for `*` symbol. {key}"
-        for i, v in enumerate(x):
-            k = key[1:].format(i=i)
-            variables[k] = v
-    elif ',' in key:
-        keys1, keys2 = [], []
-        args = None
-        for k in key.split(','):
-            if k.startswith('*'):
-                if args is None:
-                    args = k
-                else:
-                    raise ValueError(f'Only one `*` symbol is allowed. {key}')
-            elif args is None:
-                keys1.append(k)
-            else:
-                keys2.append(k)
-        assert isinstance(
-            x,
-            (list, tuple,
-             np.ndarray)), f"Should promise a list for multiple symbols. {key}"
-        if args is None:
-            assert len(keys1) == len(
-                x), f"Length of keys and values should be equal. {key}"
-            for k, v in zip(keys1, x):
-                variables[k] = v
-        else:
-            assert len(keys1) + len(keys2) <= len(
-                x), f"Too many values for unpacking. {key}"
-            for k, v in zip(keys1, x[:len(keys1)]):
-                variables[k] = v
-            end = -len(keys2) if keys2 else None
-            for i, v in enumerate(x[len(keys1):end]):
-                k = args[1:].format(i=i)
-                variables[k] = v
-            if keys2:
-                for k, v in zip(keys2, x[end:]):
-                    variables[k] = v
-    else:
-        return
-    del variables[key]
+    return description
+    return description

qulab/scan/server.py

@@ -20,6 +20,7 @@ from .models import Cell, Notebook
 from .models import Record as RecordInDB
 from .models import Session, create_engine, create_tables, sessionmaker, utcnow
 from .record import BufferList, Record, random_path
+from .utils import dump_dict, load_dict
 
 try:
     default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
@@ -53,6 +54,16 @@ class Request():
         return f"Request({self.method})"
 
 
+class Response():
+    pass
+
+
+class ErrorResponse(Response):
+
+    def __init__(self, error):
+        self.error = error
+
+
 async def reply(req, resp):
     await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
 
@@ -62,14 +73,15 @@ def clear_cache():
         return
 
     logger.debug(f"clear_cache record_cache: {len(record_cache)}")
-    for k, (t, _) in zip(sorted(record_cache.items(), key=lambda x: x[1][0]),
-                         range(len(record_cache) - CACHE_SIZE)):
+    for (k, (t, r),
+         i) in zip(sorted(record_cache.items(), key=lambda x: x[1][0]),
+                   range(len(record_cache) - CACHE_SIZE)):
         del record_cache[k]
 
     logger.debug(f"clear_cache buffer_list_cache: {len(buffer_list_cache)}")
-    for k, (t,
-            _) in zip(sorted(buffer_list_cache.items(), key=lambda x: x[1][0]),
-                      range(len(buffer_list_cache) - CACHE_SIZE)):
+    for (k, (t, r),
+         i) in zip(sorted(buffer_list_cache.items(), key=lambda x: x[1][0]),
+                   range(len(buffer_list_cache) - CACHE_SIZE)):
         del buffer_list_cache[k]
     logger.debug(f"clear_cache done.")
 
@@ -173,7 +185,7 @@ def record_delete(session: Session, record_id: int, datapath: Path):
     session.commit()
 
 
-@logger.catch
+@logger.catch(reraise=True)
 async def handle(session: Session, request: Request, datapath: Path):
 
     msg = request.msg
@@ -222,7 +234,7 @@ async def handle(session: Session, request: Request, datapath: Path):
             logger.debug(f"end bufferlist_iter_exit: {msg}")
         case 'record_create':
             logger.debug(f"record_create")
-            description = dill.loads(msg['description'])
+            description = load_dict(msg['description'])
             await reply(request, record_create(session, description, datapath))
             logger.debug(f"reply record_create")
         case 'record_append':
@@ -349,29 +361,16 @@ async def handle_with_timeout(session: Session, request: Request,
             f"Task handling request {request} timed out and was cancelled.")
         await reply(request, 'timeout')
     except Exception as e:
-        await reply(request, f'{e!r}')
+        logger.error(f"Task handling request {request} failed: {e!r}")
+        await reply(request, ErrorResponse(f'{e!r}'))
+    logger.debug(f"Task handling request {request} finished.")
 
 
 async def serv(port,
                datapath,
                url='',
                buffer_size=1024 * 1024 * 1024,
-               interval=60,
-               log='stderr',
-               debug=False):
-    if debug:
-        level = 'DEBUG'
-    else:
-        level = 'INFO'
-
-    if log == 'stderr':
-        pass
-        #logger.add(sys.stderr, level=level)
-    elif log == 'stdout':
-        logger.add(sys.stdout, level=level)
-    else:
-        logger.add(log, level=level)
-
+               interval=60):
     logger.debug('Creating socket...')
     async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
         logger.info(f'Server started at port {port}.')
@@ -392,8 +391,11 @@ async def serv(port,
                 received += len(msg)
                 try:
                     req = Request(sock, identity, msg)
-                except:
+                except Exception as e:
                     logger.exception('bad request')
+                    await sock.send_multipart(
+                        [identity,
+                         pickle.dumps(ErrorResponse(f'{e!r}'))])
                     continue
                 asyncio.create_task(
                     handle_with_timeout(session, req, datapath,
@@ -415,9 +417,21 @@ async def main(port,
                no_watch=True,
                debug=False):
     if no_watch:
+        logger.remove()
+        if debug:
+            level = 'DEBUG'
+        else:
+            level = 'INFO'
+        if log == 'stderr':
+            logger.add(sys.stderr, level=level)
+        elif log == 'stdout':
+            logger.add(sys.stdout, level=level)
+        else:
+            logger.add(sys.stderr, level=level)
+            logger.add(log, level=level)
+        logger.debug(f"logging level: {level}")
         logger.info('Server starting...')
-        await serv(port, datapath, url, buffer * 1024 * 1024, interval, log,
-                   debug)
+        await serv(port, datapath, url, buffer * 1024 * 1024, interval)
     else:
         process = None
 
@@ -444,10 +458,24 @@ async def main(port,
                         f'killed process. PID={process.pid}, returncode={process.returncode}'
                     )
                 cmd = [
-                    sys.executable, "-m", "qulab", "server", "--port",
-                    f"{port}", "--datapath", f"{datapath}", "--url", f"{url}",
-                    "--timeout", f"{timeout}", "--buffer", f"{buffer}",
-                    "--interval", f"{interval}", "--log", f"{log}",
+                    sys.executable,
+                    "-m",
+                    "qulab",
+                    "server",
+                    "--port",
+                    f"{port}",
+                    "--datapath",
+                    f"{datapath}",
+                    "--url",
+                    f"{url}",
+                    "--timeout",
+                    f"{timeout}",
+                    "--buffer",
+                    f"{buffer}",
+                    "--interval",
+                    f"{interval}",
+                    "--log",
+                    f"{log}",
                 ]
                 if url:
                     cmd.extend(['--url', url])
@@ -455,10 +483,7 @@ async def main(port,
                     cmd.append('--debug')
                 cmd.append("--no-watch")
                 logger.debug(f"starting process: {' '.join(cmd)}")
-                process = subprocess.Popen(cmd,
-                                           stdout=subprocess.PIPE,
-                                           stderr=subprocess.PIPE,
-                                           cwd=os.getcwd())
+                process = subprocess.Popen(cmd, cwd=os.getcwd())
                 logger.debug(
                     f'process started. PID={process.pid}, returncode={process.returncode}'
                 )
@@ -490,9 +515,15 @@ async def main(port,
 @click.option('--debug', is_flag=True, help='Debug mode.')
 def server(port, datapath, url, timeout, buffer, interval, log, no_watch,
            debug):
-    asyncio.run(
-        main(port, Path(datapath), url, timeout, buffer, interval, log,
-             True, debug))
+    try:
+        import uvloop
+        uvloop.run(
+            main(port, Path(datapath), url, timeout, buffer, interval, log,
+                 True, debug))
+    except ImportError:
+        asyncio.run(
+            main(port, Path(datapath), url, timeout, buffer, interval, log,
+                 True, debug))
 
 
 if __name__ == "__main__":

qulab/scan/space.py

@@ -25,12 +25,27 @@ class Space():
 
     @classmethod
     def fromarray(cls, space):
-        if isinstance(space, Space):
+        if isinstance(space, (Space, range, enumerate, tuple)):
             return space
-        if isinstance(space, (list, tuple)):
-            array = np.array(space)
+        if isinstance(space, list):
+            if isinstance(space[0], int):
+                try:
+                    if all(i == j for i, j in zip(
+                            space,
+                            range(space[0], space[-1] + 1, space[1] -
+                                  space[0]))):
+                        return range(space[0], space[-1] + 1,
+                                     space[1] - space[0])
+                except:
+                    return space
+            elif isinstance(space[0], (float, complex, np.ndarray)):
+                array = np.array(space)
+            else:
+                return space
         elif isinstance(space, np.ndarray):
             array = space
+        else:
+            return space
         try:
             a = np.linspace(array[0], array[-1], len(array), dtype=array.dtype)
             if np.allclose(a, array):

qulab/scan/utils.py

@@ -1,6 +1,11 @@
 import ast
 import asyncio
 import inspect
+import platform
+import re
+import subprocess
+import sys
+import uuid
 import warnings
 from typing import Any, Callable
 
@@ -29,6 +34,46 @@ class TooLarge:
         return f'<TooLarge: {self.type} at 0x{id(self):x}>'
 
 
+def dump_dict(d, keys=[]):
+    ret = {}
+
+    for key, value in d.items():
+        if key in keys:
+            ret[key] = value
+            continue
+        if isinstance(value, dict) and isinstance(key, str):
+            ret[key] = dump_dict(value,
+                                 keys=[
+                                     k[len(key) + 1:] for k in keys
+                                     if k.startswith(f'{key}.')
+                                 ])
+        else:
+            try:
+                ret[key] = dill.dumps(value)
+            except:
+                ret[key] = Unpicklable(value)
+
+    return dill.dumps(ret)
+
+
+def load_dict(buff):
+    if isinstance(buff, dict):
+        return {key: load_dict(value) for key, value in buff.items()}
+
+    if not isinstance(buff, bytes):
+        return buff
+
+    try:
+        ret = dill.loads(buff)
+    except:
+        return buff
+
+    if isinstance(ret, dict):
+        return load_dict(ret)
+    else:
+        return ret
+
+
 def dump_globals(ns=None, *, size_limit=10 * 1024 * 1024, warn=False):
     import __main__
 
@@ -129,3 +174,56 @@ async def call_function(func: Callable | Expression, variables: dict[str,
     if inspect.isawaitable(ret):
         ret = await ret
     return ret
+
+
+def yapf_reformat(cell_text):
+    try:
+        import isort
+        import yapf.yapflib.yapf_api
+
+        fname = f"f{uuid.uuid1().hex}"
+
+        def wrap(source):
+            lines = [f"async def {fname}():"]
+            for line in source.split('\n'):
+                lines.append("    " + line)
+            return '\n'.join(lines)
+
+        def unwrap(source):
+            lines = []
+            for line in source.split('\n'):
+                if line.startswith(f"async def {fname}():"):
+                    continue
+                lines.append(line[4:])
+            return '\n'.join(lines)
+
+        cell_text = re.sub('^%', '#%#', cell_text, flags=re.M)
+        reformated_text = unwrap(
+            yapf.yapflib.yapf_api.FormatCode(wrap(isort.code(cell_text)))[0])
+        return re.sub('^#%#', '%', reformated_text, flags=re.M)
+    except:
+        return cell_text
+
+
+def get_installed_packages():
+    result = subprocess.run([sys.executable, '-m', 'pip', 'freeze'],
+                            stdout=subprocess.PIPE,
+                            text=True)
+
+    lines = result.stdout.split('\n')
+    packages = []
+    for line in lines:
+        if line:
+            packages.append(line)
+    return packages
+
+
+def get_system_info():
+    info = {
+        'OS': platform.uname()._asdict(),
+        'Python': sys.version,
+        'PythonExecutable': sys.executable,
+        'PythonPath': sys.path,
+        'packages': get_installed_packages()
+    }
+    return info

qulab/version.py

@@ -1 +1 @@
-__version__ = "2.3.3"
+__version__ = "2.3.5"