QuLab 2.3.3__cp312-cp312-win_amd64.whl → 2.3.4__cp312-cp312-win_amd64.whl

{QuLab-2.3.3.dist-info → QuLab-2.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.3.3
+Version: 2.3.4
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>
@@ -38,6 +38,9 @@ Requires-Dist: ply >=3.11
 Requires-Dist: pyzmq >=25.1.0
 Requires-Dist: scipy >=1.0.0
 Requires-Dist: watchdog >=4.0.0
+Provides-Extra: full
+Requires-Dist: SQLAlchemy >=2.0.19 ; extra == 'full'
+Requires-Dist: uvloop >=0.19.0 ; extra == 'full'
 
 # QuLab
 [![View build status](https://travis-ci.org/feihoo87/QuLab.svg?branch=master)](https://travis-ci.org/feihoo87/QuLab)

{QuLab-2.3.3.dist-info → QuLab-2.3.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=vkFybY8YSsQilYdThPRD83-btPAR41sy_WCXiM-6mME,141
 qulab/__main__.py,sha256=V7iokU7awstgjCeiF_hoOdFyrqJwC_4QetiLe7cWvOQ,454
-qulab/fun.cp312-win_amd64.pyd,sha256=9TAJFGcYh6nLJr2XAcpOCuv2qhxMcO9OAxXizK83um4,32256
-qulab/version.py,sha256=_TojZ2sN-5aiITSzim-_br5UuW9pwpo2Tx8vnDWpsqE,21
+qulab/fun.cp312-win_amd64.pyd,sha256=cVkHz7_RIsRL4Ij6Ku9j5XNMr2PPk6iNBG1kxgNB528,32256
+qulab/version.py,sha256=MtmOJMh3yHY9aTdRsZDgbpPfbzm9xdycdi9jjMKzYjM,21
 qulab/monitor/__init__.py,sha256=xEVDkJF8issrsDeLqQmDsvtRmrf-UiViFcGTWuzdlFU,43
 qulab/monitor/__main__.py,sha256=k2H1H5Zf9LLXTDLISJkbikLH-z0f1e5i5i6wXXYPOrE,105
 qulab/monitor/config.py,sha256=y_5StMkdrbZO1ziyKBrvIkB7Jclp9RCPK1QbsOhCxnY,785
@@ -19,10 +19,10 @@ qulab/scan/models.py,sha256=ZvXkJEt5Yz3Sjx0JKzYka-q2Uo-w_iVzHgH8A6DbjF0,18236
 qulab/scan/optimize.py,sha256=ACfGSfrFpPgmZo_P5kD4mquahXImYipdP7E86dtFQO8,2635
 qulab/scan/query.py,sha256=RM8bG4Tcx_PaNk8tv9HdlTZ1dGuuSr3sZVkYVq2BtfQ,12183
 qulab/scan/record.py,sha256=MVmxhIzwmOju7eWxJEWsqJZlVgrDeRXGMfNvXImj7Ms,21883
-qulab/scan/scan.py,sha256=HMeVxoaEWjOj0eBusrQ-QQUFDAc_ivoLv95XkDmOS70,39947
-qulab/scan/server.py,sha256=em2tM_JPySnUNhusvl92ZPUGxz1mM34jFo5v2ji4uy0,19611
+qulab/scan/scan.py,sha256=PDU0oZkwCntucOM4aNNgY22t-Ng8XChmFrhTqPqxDX8,41814
+qulab/scan/server.py,sha256=ahdVpTKm8P2qB7E9HKM8b95HF89YlCJ5WWLmdJhMfgw,20479
 qulab/scan/space.py,sha256=S-jHaXXf12FzjajOhPsZ-mkpGzAxrFAI3tRuQrvTBKg,6274
-qulab/scan/utils.py,sha256=30qnYvyFyctwcWxOCUpCNxXgGysA7xdIDzYbjwxGUzA,3744
+qulab/scan/utils.py,sha256=j5ai6E-pK2tYEPYlL1FpgHG7q2DL1IVAa-hPPgz5SlQ,4757
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=6-EjN0waX1yfcMPJXqpIr9UlrIEsSCFApm5G-ZeaPMQ,1742
 qulab/storage/base_dataset.py,sha256=28y3-OZrqJ52p5sbirEpUgjb7hqwLLpd38KU9DCkD24,12217
@@ -78,9 +78,9 @@ qulab/visualization/plot_layout.py,sha256=yAnMONOms7_szCdng-8wPpUMPis5UnbaNNzV4K
 qulab/visualization/plot_seq.py,sha256=h9D0Yl_yO64IwlvBgzMu9EBKr9gg6y8QE55gu2PfTns,2783
 qulab/visualization/qdat.py,sha256=HubXFu4nfcA7iUzghJGle1C86G6221hicLR0b-GqhKQ,5887
 qulab/visualization/widgets.py,sha256=HcYwdhDtLreJiYaZuN3LfofjJmZcLwjMfP5aasebgDo,3266
-QuLab-2.3.3.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
-QuLab-2.3.3.dist-info/METADATA,sha256=jKqntTp7yF3mnpI4ItBA9tLV0zjlI-v_uRTrf_AonRk,3609
-QuLab-2.3.3.dist-info/WHEEL,sha256=h3VxxLkZstcnkT86k1ujrti5ToMTN-RbBnzIRuTWCMs,101
-QuLab-2.3.3.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.3.3.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.3.3.dist-info/RECORD,,
+QuLab-2.3.4.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
+QuLab-2.3.4.dist-info/METADATA,sha256=t-ZisaizQKo8edK0p_-vQD8ljsuXo9cl2dg5wE-sFsU,3735
+QuLab-2.3.4.dist-info/WHEEL,sha256=h3VxxLkZstcnkT86k1ujrti5ToMTN-RbBnzIRuTWCMs,101
+QuLab-2.3.4.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.3.4.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.3.4.dist-info/RECORD,,

qulab/scan/scan.py

@@ -26,7 +26,7 @@ from .optimize import NgOptimizer
 from .record import Record
 from .server import default_record_port
 from .space import Optimizer, OptimizeSpace, Space
-from .utils import async_zip, call_function, dump_globals
+from .utils import async_zip, call_function, dump_dict, dump_globals
 
 try:
     from tqdm.notebook import tqdm
@@ -205,19 +205,6 @@ def _run_function_in_process(buf):
     return func(*args, **kwds)
 
 
-def _dump_description(description):
-    d = {}
-    for key, value in description.items():
-        if key in [
-                'intrinsic_loops', 'app', 'tags', 'loops',
-                'independent_variables', 'axis', 'config', 'entry'
-        ]:
-            d[key] = value
-        else:
-            d[key] = dill.dumps(value)
-    return dill.dumps(description)
-
-
 class Scan():
 
     def __new__(cls, *args, mixin=None, **kwds):
@@ -277,6 +264,7 @@ class Scan():
         self._current_level = 0
         self._variables = {}
         self._main_task = None
+        self._background_tasks = ()
         self._sock = None
         self._sem = asyncio.Semaphore(max_promise + 1)
         self._bar: dict[int, tqdm] = {}
@@ -300,6 +288,7 @@ class Scan():
         del state['record']
         del state['_sock']
         del state['_main_task']
+        del state['_background_tasks']
         del state['_bar']
         del state['_msg_queue']
         del state['_prm_queue']
@@ -312,6 +301,7 @@ class Scan():
         self.record = None
         self._sock = None
         self._main_task = None
+        self._background_tasks = ()
         self._bar = {}
         self._prm_queue = asyncio.Queue()
         self._msg_queue = asyncio.Queue(self._max_message)
@@ -412,7 +402,11 @@ class Scan():
             'method':
             'record_create',
             'description':
-            _dump_description(self.description)
+            dump_dict(self.description,
+                      keys=[
+                          'intrinsic_loops', 'app', 'tags', 'loops',
+                          'independent_variables', 'axis', 'config', 'entry'
+                      ])
         })
 
         record_id = await self._sock.recv_pyobj()
@@ -612,27 +606,33 @@ class Scan():
             await task
             self._msg_queue.task_done()
 
+    @contextlib.asynccontextmanager
+    async def _send_msg_and_update_bar(self):
+        send_msg_task = asyncio.create_task(self._send_msg())
+        update_progress_task = asyncio.create_task(self._update_progress())
+        try:
+            yield (send_msg_task, update_progress_task)
+        finally:
+            update_progress_task.cancel()
+            send_msg_task.cancel()
+            while True:
+                try:
+                    task = self._prm_queue.get_nowait()
+                except:
+                    break
+                try:
+                    task.cancel()
+                except:
+                    pass
+
+    async def _check_background_tasks(self):
+        for task in self._background_tasks:
+            if task.done():
+                await task
+
     async def run(self):
         assymbly(self.description)
-
-        @contextlib.asynccontextmanager
-        async def send_msg_and_update_bar(self):
-            send_msg_task = asyncio.create_task(self._send_msg())
-            update_progress_task = asyncio.create_task(self._update_progress())
-            try:
-                yield
-            finally:
-                update_progress_task.cancel()
-                send_msg_task.cancel()
-                while True:
-                    try:
-                        task = self._prm_queue.get_nowait()
-                    except:
-                        break
-                    try:
-                        task.cancel()
-                    except:
-                        pass
+        self._background_tasks = ()
 
         if isinstance(
                 self.description['database'],
@@ -641,12 +641,14 @@ class Scan():
                                          connect=self.description['database'],
                                          socket=self._sock) as socket:
                 self._sock = socket
-                async with send_msg_and_update_bar(self):
+                async with self._send_msg_and_update_bar() as background_tasks:
+                    self._background_tasks = background_tasks
                     await self._run()
         else:
             if self.config:
                 self.description['config'] = copy.deepcopy(self.config)
-            async with send_msg_and_update_bar(self):
+            async with self._send_msg_and_update_bar() as background_tasks:
+                self._background_tasks = background_tasks
                 await self._run()
 
     async def _run(self):
@@ -671,11 +673,11 @@ class Scan():
             self.description['functions'], self.variables)
         await update_variables(self.variables, updates,
                                self.description['setters'])
-
+        await self._check_background_tasks()
         await self.work()
         for level, bar in self._bar.items():
             bar.close()
-
+        await self._check_background_tasks()
         if self._single_step:
             self.variables.update(await call_many_functions(
                 self.description['order'].get(-1, []),
@@ -683,7 +685,7 @@ class Scan():
 
             await self.emit(0, 0, 0, self.variables)
             await self.emit(-1, 0, 0, {})
-
+        await self._check_background_tasks()
         await self._prm_queue.join()
         await self._msg_queue.join()
         return self.variables
@@ -747,6 +749,7 @@ class Scan():
                 | {'config': self._synchronize_config},
                 self.description['optimizers'], self.description['setters'],
                 self.description['getters']):
+            await self._check_background_tasks()
             self._current_level += 1
             if await self._filter(variables, self.current_level - 1):
                 yield variables
@@ -758,11 +761,13 @@ class Scan():
             self._current_level -= 1
             self._prm_queue.put_nowait(
                 self._update_progress_bar(self.current_level, 1))
+            await self._check_background_tasks()
         if self.current_level == 0:
             await self.emit(self.current_level - 1, 0, 0, {})
             for name, value in self.variables.items():
                 if inspect.isawaitable(value):
                     self.variables[name] = await value
+            await self._check_background_tasks()
             await self._prm_queue.join()
 
     async def work(self, **kwds):
@@ -823,7 +828,7 @@ class Scan():
             return await awaitable
 
 
-def assymbly(description):
+def _get_environment(description):
     import __main__
     from IPython import get_ipython
 
@@ -853,6 +858,10 @@ def assymbly(description):
 
     description['entry']['env'] = {k: v for k, v in os.environ.items()}
 
+    return description
+
+
+def _mapping_levels(description):
     mapping = {
         label: level
         for level, label in enumerate(
@@ -885,8 +894,23 @@ def assymbly(description):
                                                   len(space))
             except:
                 pass
+    return levels
+
+
+def _get_independent_variables(description):
+    independent_variables = set(description['intrinsic_loops'].keys())
+    for level, loops in description['loops'].items():
+        for name, iterable in loops:
+            if isinstance(iterable, (np.ndarray, list, tuple, range, Space)):
+                independent_variables.add(name)
+    return independent_variables
+
 
+def _build_dependents(description, levels, independent_variables):
     dependents = copy.deepcopy(description['dependents'])
+    all_nodes = set(description['dependents'].keys())
+    for key, deps in dependents.items():
+        all_nodes.update(deps)
 
     for level in levels:
         range_list = description['loops'].get(level, [])
@@ -899,6 +923,13 @@ def assymbly(description):
                 dependents[name] = set()
             dependents[name].add(f'#__loop_{level}')
 
+    after_yield = set()
+    for key in all_nodes:
+        if key not in independent_variables and key not in description[
+                'consts']:
+            if key not in dependents:
+                after_yield.add(key)
+
     def _get_all_depends(key, graph):
         ret = set()
         if key not in graph:
@@ -912,7 +943,13 @@ def assymbly(description):
     full_depends = {}
     for key in dependents:
         full_depends[key] = _get_all_depends(key, dependents)
+        if full_depends[key] & after_yield:
+            after_yield.add(key)
+
+    return dependents, full_depends, after_yield
 
+
+def _build_order(description, levels, dependents, full_depends):
     levels = {}
     passed = set()
     all_keys = set(description['consts'].keys())
@@ -957,8 +994,9 @@ def assymbly(description):
                 description['order'][level].append(ready)
                 keys -= set(ready)
 
+
+def _make_axis(description):
     axis = {}
-    independent_variables = set(description['intrinsic_loops'].keys())
 
     for name in description['consts']:
         axis[name] = ()
@@ -967,8 +1005,6 @@ def assymbly(description):
             if isinstance(iterable, OptimizeSpace):
                 axis[name] = tuple(range(level + 1))
                 continue
-            elif isinstance(iterable, (np.ndarray, list, tuple, range, Space)):
-                independent_variables.add(name)
             axis[name] = (level, )
 
     for level, group in description['order'].items():
@@ -989,8 +1025,20 @@ def assymbly(description):
         k: tuple([x for x in v if x >= 0])
         for k, v in axis.items()
     }
+
+
+def assymbly(description):
+    _get_environment(description)
+    levels = _mapping_levels(description)
+    independent_variables = _get_independent_variables(description)
     description['independent_variables'] = independent_variables
 
+    dependents, full_depends, after_yield = _build_dependents(
+        description, levels, independent_variables)
+
+    _build_order(description, levels, dependents, full_depends)
+    _make_axis(description)
+
     return description
 
 

qulab/scan/server.py

@@ -20,6 +20,7 @@ from .models import Cell, Notebook
 from .models import Record as RecordInDB
 from .models import Session, create_engine, create_tables, sessionmaker, utcnow
 from .record import BufferList, Record, random_path
+from .utils import dump_dict, load_dict
 
 try:
     default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
@@ -53,6 +54,16 @@ class Request():
         return f"Request({self.method})"
 
 
+class Response():
+    pass
+
+
+class ErrorResponse(Response):
+
+    def __init__(self, error):
+        self.error = error
+
+
 async def reply(req, resp):
     await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
 
@@ -173,7 +184,7 @@ def record_delete(session: Session, record_id: int, datapath: Path):
     session.commit()
 
 
-@logger.catch
+@logger.catch(reraise=True)
 async def handle(session: Session, request: Request, datapath: Path):
 
     msg = request.msg
@@ -222,7 +233,7 @@ async def handle(session: Session, request: Request, datapath: Path):
             logger.debug(f"end bufferlist_iter_exit: {msg}")
         case 'record_create':
             logger.debug(f"record_create")
-            description = dill.loads(msg['description'])
+            description = load_dict(msg['description'])
             await reply(request, record_create(session, description, datapath))
             logger.debug(f"reply record_create")
         case 'record_append':
@@ -349,29 +360,16 @@ async def handle_with_timeout(session: Session, request: Request,
             f"Task handling request {request} timed out and was cancelled.")
         await reply(request, 'timeout')
     except Exception as e:
-        await reply(request, f'{e!r}')
+        logger.error(f"Task handling request {request} failed: {e!r}")
+        await reply(request, ErrorResponse(f'{e!r}'))
+    logger.debug(f"Task handling request {request} finished.")
 
 
 async def serv(port,
                datapath,
                url='',
                buffer_size=1024 * 1024 * 1024,
-               interval=60,
-               log='stderr',
-               debug=False):
-    if debug:
-        level = 'DEBUG'
-    else:
-        level = 'INFO'
-
-    if log == 'stderr':
-        pass
-        #logger.add(sys.stderr, level=level)
-    elif log == 'stdout':
-        logger.add(sys.stdout, level=level)
-    else:
-        logger.add(log, level=level)
-
+               interval=60):
     logger.debug('Creating socket...')
     async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
         logger.info(f'Server started at port {port}.')
@@ -392,8 +390,11 @@ async def serv(port,
                 received += len(msg)
                 try:
                     req = Request(sock, identity, msg)
-                except:
+                except Exception as e:
                     logger.exception('bad request')
+                    await sock.send_multipart(
+                        [identity,
+                         pickle.dumps(ErrorResponse(f'{e!r}'))])
                     continue
                 asyncio.create_task(
                     handle_with_timeout(session, req, datapath,
@@ -415,9 +416,21 @@ async def main(port,
                no_watch=True,
                debug=False):
     if no_watch:
+        logger.remove()
+        if debug:
+            level = 'DEBUG'
+        else:
+            level = 'INFO'
+        if log == 'stderr':
+            logger.add(sys.stderr, level=level)
+        elif log == 'stdout':
+            logger.add(sys.stdout, level=level)
+        else:
+            logger.add(sys.stderr, level=level)
+            logger.add(log, level=level)
+        logger.debug(f"logging level: {level}")
         logger.info('Server starting...')
-        await serv(port, datapath, url, buffer * 1024 * 1024, interval, log,
-                   debug)
+        await serv(port, datapath, url, buffer * 1024 * 1024, interval)
     else:
         process = None
 
@@ -444,10 +457,24 @@ async def main(port,
                         f'killed process. PID={process.pid}, returncode={process.returncode}'
                     )
                 cmd = [
-                    sys.executable, "-m", "qulab", "server", "--port",
-                    f"{port}", "--datapath", f"{datapath}", "--url", f"{url}",
-                    "--timeout", f"{timeout}", "--buffer", f"{buffer}",
-                    "--interval", f"{interval}", "--log", f"{log}",
+                    sys.executable,
+                    "-m",
+                    "qulab",
+                    "server",
+                    "--port",
+                    f"{port}",
+                    "--datapath",
+                    f"{datapath}",
+                    "--url",
+                    f"{url}",
+                    "--timeout",
+                    f"{timeout}",
+                    "--buffer",
+                    f"{buffer}",
+                    "--interval",
+                    f"{interval}",
+                    "--log",
+                    f"{log}",
                 ]
                 if url:
                     cmd.extend(['--url', url])
@@ -455,10 +482,7 @@ async def main(port,
                     cmd.append('--debug')
                 cmd.append("--no-watch")
                 logger.debug(f"starting process: {' '.join(cmd)}")
-                process = subprocess.Popen(cmd,
-                                           stdout=subprocess.PIPE,
-                                           stderr=subprocess.PIPE,
-                                           cwd=os.getcwd())
+                process = subprocess.Popen(cmd, cwd=os.getcwd())
                 logger.debug(
                     f'process started. PID={process.pid}, returncode={process.returncode}'
                 )
@@ -490,9 +514,15 @@ async def main(port,
 @click.option('--debug', is_flag=True, help='Debug mode.')
 def server(port, datapath, url, timeout, buffer, interval, log, no_watch,
            debug):
-    asyncio.run(
-        main(port, Path(datapath), url, timeout, buffer, interval, log,
-             True, debug))
+    try:
+        import uvloop
+        uvloop.run(
+            main(port, Path(datapath), url, timeout, buffer, interval, log,
+                 True, debug))
+    except ImportError:
+        asyncio.run(
+            main(port, Path(datapath), url, timeout, buffer, interval, log,
+                 True, debug))
 
 
 if __name__ == "__main__":

qulab/scan/utils.py

@@ -29,6 +29,46 @@ class TooLarge:
         return f'<TooLarge: {self.type} at 0x{id(self):x}>'
 
 
+def dump_dict(d, keys=[]):
+    ret = {}
+
+    for key, value in d.items():
+        if key in keys:
+            ret[key] = value
+            continue
+        if isinstance(value, dict) and isinstance(key, str):
+            ret[key] = dump_dict(value,
+                                 keys=[
+                                     k[len(key) + 1:] for k in keys
+                                     if k.startswith(f'{key}.')
+                                 ])
+        else:
+            try:
+                ret[key] = dill.dumps(value)
+            except:
+                ret[key] = Unpicklable(value)
+
+    return dill.dumps(ret)
+
+
+def load_dict(buff):
+    if isinstance(buff, dict):
+        return {key: load_dict(value) for key, value in buff.items()}
+
+    if not isinstance(buff, bytes):
+        return buff
+
+    try:
+        ret = dill.loads(buff)
+    except:
+        return buff
+
+    if isinstance(ret, dict):
+        return load_dict(ret)
+    else:
+        return ret
+
+
 def dump_globals(ns=None, *, size_limit=10 * 1024 * 1024, warn=False):
     import __main__
 

qulab/version.py

@@ -1 +1 @@
-__version__ = "2.3.3"
+__version__ = "2.3.4"