QuLab 2.3.3__cp310-cp310-macosx_10_9_universal2.whl → 2.3.4__cp310-cp310-macosx_10_9_universal2.whl

{QuLab-2.3.3.dist-info → QuLab-2.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.3.3
+Version: 2.3.4
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>
@@ -38,6 +38,9 @@ Requires-Dist: ply >=3.11
 Requires-Dist: pyzmq >=25.1.0
 Requires-Dist: scipy >=1.0.0
 Requires-Dist: watchdog >=4.0.0
+Provides-Extra: full
+Requires-Dist: SQLAlchemy >=2.0.19 ; extra == 'full'
+Requires-Dist: uvloop >=0.19.0 ; extra == 'full'
 
 # QuLab
 [![View build status](https://travis-ci.org/feihoo87/QuLab.svg?branch=master)](https://travis-ci.org/feihoo87/QuLab)

{QuLab-2.3.3.dist-info → QuLab-2.3.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=P-Mx2p4TVmL91SoxoeXcj8Qm0x4xUf5Q_FLk0Yc_gIQ,138
 qulab/__main__.py,sha256=ZC1NKaoxKyy60DaCfB8vYnB1z3RXQ2j8E1sRZ4A8sXE,428
-qulab/fun.cpython-310-darwin.so,sha256=EOGb46gUpfcUMp6umfTCrC5pq-XWOx8aVISFT_8OKrg,159632
-qulab/version.py,sha256=_TojZ2sN-5aiITSzim-_br5UuW9pwpo2Tx8vnDWpsqE,21
+qulab/fun.cpython-310-darwin.so,sha256=Sr82tYA3bW2CSKKXnJaWdMqPV85avNqZnKlzFD-98aE,159632
+qulab/version.py,sha256=MtmOJMh3yHY9aTdRsZDgbpPfbzm9xdycdi9jjMKzYjM,21
 qulab/monitor/__init__.py,sha256=nTHelnDpxRS_fl_B38TsN0njgq8eVTEz9IAnN3NbDlM,42
 qulab/monitor/__main__.py,sha256=w3yUcqq195LzSnXTkQcuC1RSFRhy4oQ_PEBmucXguME,97
 qulab/monitor/config.py,sha256=fQ5JcsMApKc1UwANEnIvbDQZl8uYW0tle92SaYtX9lI,744
@@ -19,10 +19,10 @@ qulab/scan/models.py,sha256=5Jpo25WGMWs0GtLzYLsWO61G3-FFYx5BHhBr2b6rOTE,17681
 qulab/scan/optimize.py,sha256=jYceGazEabuwYqmZE8P6Nnq_KsLyUP0p-p88VbbJLhM,2559
 qulab/scan/query.py,sha256=-5uHMhXSyGovK1oy_uUbGVEbRFzaMBkP78ZMNfI3jD0,11809
 qulab/scan/record.py,sha256=yIHPANf6nuBXy8Igf-dMtGJ7wuFTLYlBaaAUc0AzwyU,21280
-qulab/scan/scan.py,sha256=GhRhBLWb5PVFi9qkyXJl0p1XXbpjdnjoSz_wPegf9Vs,38783
-qulab/scan/server.py,sha256=tGQ4bVePqvFACb0L26viA57UBD7LkkZI9q5kQGffapA,19112
+qulab/scan/scan.py,sha256=hrCnoIROMPoYXEZnU9GTZUR-aX9qP58d40wlN3DTQCE,40602
+qulab/scan/server.py,sha256=_4YrOJcnqDTnxryZXE8lEKGzuT4LS-8q9bN6jrTEqUQ,19950
 qulab/scan/space.py,sha256=X5anTvw7X0XGN83y0o0C-zhvh6bDEU-kshGgo004pNE,6076
-qulab/scan/utils.py,sha256=Pg_tCf3SUKTiPSBqb6Enkgx4bAyQJAkDGe9uYys1xVU,3613
+qulab/scan/utils.py,sha256=zCV_Td8hBgb16Hv3XKo64KYDDpvBw2rQ83fxq1hvjG8,4586
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=3emxxRry8BB0m8hUZvJ_oBqkPy7ksV7flHB_KEDXZuI,1692
 qulab/storage/base_dataset.py,sha256=4aKhqBNdZfdlm_z1Qy5dv0HrvgpvMdy8pbyfua8uE-4,11865
@@ -78,9 +78,9 @@ qulab/visualization/plot_layout.py,sha256=clNw9QjE_kVNpIIx2Ob4YhAz2fucPGMuzkoIrO
 qulab/visualization/plot_seq.py,sha256=lphYF4VhkEdc_wWr1kFBwrx2yujkyFPFaJ3pjr61awI,2693
 qulab/visualization/qdat.py,sha256=ZeevBYWkzbww4xZnsjHhw7wRorJCBzbG0iEu-XQB4EA,5735
 qulab/visualization/widgets.py,sha256=6KkiTyQ8J-ei70LbPQZAK35wjktY47w2IveOa682ftA,3180
-QuLab-2.3.3.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
-QuLab-2.3.3.dist-info/METADATA,sha256=DcGCVF8ABDW4sJ8mRhZ2N8jWlYXRA3yvHV6sLt_T24o,3510
-QuLab-2.3.3.dist-info/WHEEL,sha256=ClVOi9g10rI0HLG8M8rsO0ZIZ1smbNYM10V9YDSXCJY,114
-QuLab-2.3.3.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.3.3.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.3.3.dist-info/RECORD,,
+QuLab-2.3.4.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
+QuLab-2.3.4.dist-info/METADATA,sha256=Noz4IplxqE9TBtcN2Qr38B1J0vtpEkCWsjAjvRMGF_I,3633
+QuLab-2.3.4.dist-info/WHEEL,sha256=ClVOi9g10rI0HLG8M8rsO0ZIZ1smbNYM10V9YDSXCJY,114
+QuLab-2.3.4.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.3.4.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.3.4.dist-info/RECORD,,

qulab/scan/scan.py

@@ -26,7 +26,7 @@ from .optimize import NgOptimizer
 from .record import Record
 from .server import default_record_port
 from .space import Optimizer, OptimizeSpace, Space
-from .utils import async_zip, call_function, dump_globals
+from .utils import async_zip, call_function, dump_dict, dump_globals
 
 try:
     from tqdm.notebook import tqdm
@@ -205,19 +205,6 @@ def _run_function_in_process(buf):
     return func(*args, **kwds)
 
 
-def _dump_description(description):
-    d = {}
-    for key, value in description.items():
-        if key in [
-                'intrinsic_loops', 'app', 'tags', 'loops',
-                'independent_variables', 'axis', 'config', 'entry'
-        ]:
-            d[key] = value
-        else:
-            d[key] = dill.dumps(value)
-    return dill.dumps(description)
-
-
 class Scan():
 
     def __new__(cls, *args, mixin=None, **kwds):
@@ -277,6 +264,7 @@ class Scan():
         self._current_level = 0
         self._variables = {}
         self._main_task = None
+        self._background_tasks = ()
         self._sock = None
         self._sem = asyncio.Semaphore(max_promise + 1)
         self._bar: dict[int, tqdm] = {}
@@ -300,6 +288,7 @@ class Scan():
         del state['record']
         del state['_sock']
         del state['_main_task']
+        del state['_background_tasks']
         del state['_bar']
         del state['_msg_queue']
         del state['_prm_queue']
@@ -312,6 +301,7 @@ class Scan():
         self.record = None
         self._sock = None
         self._main_task = None
+        self._background_tasks = ()
         self._bar = {}
         self._prm_queue = asyncio.Queue()
         self._msg_queue = asyncio.Queue(self._max_message)
@@ -412,7 +402,11 @@ class Scan():
             'method':
             'record_create',
             'description':
-            _dump_description(self.description)
+            dump_dict(self.description,
+                      keys=[
+                          'intrinsic_loops', 'app', 'tags', 'loops',
+                          'independent_variables', 'axis', 'config', 'entry'
+                      ])
         })
 
         record_id = await self._sock.recv_pyobj()
@@ -612,27 +606,33 @@ class Scan():
             await task
             self._msg_queue.task_done()
 
+    @contextlib.asynccontextmanager
+    async def _send_msg_and_update_bar(self):
+        send_msg_task = asyncio.create_task(self._send_msg())
+        update_progress_task = asyncio.create_task(self._update_progress())
+        try:
+            yield (send_msg_task, update_progress_task)
+        finally:
+            update_progress_task.cancel()
+            send_msg_task.cancel()
+            while True:
+                try:
+                    task = self._prm_queue.get_nowait()
+                except:
+                    break
+                try:
+                    task.cancel()
+                except:
+                    pass
+
+    async def _check_background_tasks(self):
+        for task in self._background_tasks:
+            if task.done():
+                await task
+
     async def run(self):
         assymbly(self.description)
-
-        @contextlib.asynccontextmanager
-        async def send_msg_and_update_bar(self):
-            send_msg_task = asyncio.create_task(self._send_msg())
-            update_progress_task = asyncio.create_task(self._update_progress())
-            try:
-                yield
-            finally:
-                update_progress_task.cancel()
-                send_msg_task.cancel()
-                while True:
-                    try:
-                        task = self._prm_queue.get_nowait()
-                    except:
-                        break
-                    try:
-                        task.cancel()
-                    except:
-                        pass
+        self._background_tasks = ()
 
         if isinstance(
                 self.description['database'],
@@ -641,12 +641,14 @@ class Scan():
                                          connect=self.description['database'],
                                          socket=self._sock) as socket:
                 self._sock = socket
-                async with send_msg_and_update_bar(self):
+                async with self._send_msg_and_update_bar() as background_tasks:
+                    self._background_tasks = background_tasks
                     await self._run()
         else:
             if self.config:
                 self.description['config'] = copy.deepcopy(self.config)
-            async with send_msg_and_update_bar(self):
+            async with self._send_msg_and_update_bar() as background_tasks:
+                self._background_tasks = background_tasks
                 await self._run()
 
     async def _run(self):
@@ -671,11 +673,11 @@ class Scan():
             self.description['functions'], self.variables)
         await update_variables(self.variables, updates,
                                self.description['setters'])
-
+        await self._check_background_tasks()
         await self.work()
         for level, bar in self._bar.items():
             bar.close()
-
+        await self._check_background_tasks()
         if self._single_step:
             self.variables.update(await call_many_functions(
                 self.description['order'].get(-1, []),
@@ -683,7 +685,7 @@ class Scan():
 
             await self.emit(0, 0, 0, self.variables)
             await self.emit(-1, 0, 0, {})
-
+        await self._check_background_tasks()
         await self._prm_queue.join()
         await self._msg_queue.join()
         return self.variables
@@ -747,6 +749,7 @@ class Scan():
                 | {'config': self._synchronize_config},
                 self.description['optimizers'], self.description['setters'],
                 self.description['getters']):
+            await self._check_background_tasks()
             self._current_level += 1
             if await self._filter(variables, self.current_level - 1):
                 yield variables
@@ -758,11 +761,13 @@ class Scan():
             self._current_level -= 1
             self._prm_queue.put_nowait(
                 self._update_progress_bar(self.current_level, 1))
+            await self._check_background_tasks()
         if self.current_level == 0:
             await self.emit(self.current_level - 1, 0, 0, {})
             for name, value in self.variables.items():
                 if inspect.isawaitable(value):
                     self.variables[name] = await value
+            await self._check_background_tasks()
             await self._prm_queue.join()
 
     async def work(self, **kwds):
@@ -823,7 +828,7 @@ class Scan():
             return await awaitable
 
 
-def assymbly(description):
+def _get_environment(description):
     import __main__
     from IPython import get_ipython
 
@@ -853,6 +858,10 @@ def assymbly(description):
 
     description['entry']['env'] = {k: v for k, v in os.environ.items()}
 
+    return description
+
+
+def _mapping_levels(description):
     mapping = {
         label: level
         for level, label in enumerate(
@@ -885,8 +894,23 @@ def assymbly(description):
                                                   len(space))
             except:
                 pass
+    return levels
+
+
+def _get_independent_variables(description):
+    independent_variables = set(description['intrinsic_loops'].keys())
+    for level, loops in description['loops'].items():
+        for name, iterable in loops:
+            if isinstance(iterable, (np.ndarray, list, tuple, range, Space)):
+                independent_variables.add(name)
+    return independent_variables
+
 
+def _build_dependents(description, levels, independent_variables):
     dependents = copy.deepcopy(description['dependents'])
+    all_nodes = set(description['dependents'].keys())
+    for key, deps in dependents.items():
+        all_nodes.update(deps)
 
     for level in levels:
         range_list = description['loops'].get(level, [])
@@ -899,6 +923,13 @@ def assymbly(description):
                 dependents[name] = set()
             dependents[name].add(f'#__loop_{level}')
 
+    after_yield = set()
+    for key in all_nodes:
+        if key not in independent_variables and key not in description[
+                'consts']:
+            if key not in dependents:
+                after_yield.add(key)
+
     def _get_all_depends(key, graph):
         ret = set()
         if key not in graph:
@@ -912,7 +943,13 @@ def assymbly(description):
     full_depends = {}
     for key in dependents:
         full_depends[key] = _get_all_depends(key, dependents)
+        if full_depends[key] & after_yield:
+            after_yield.add(key)
+
+    return dependents, full_depends, after_yield
 
+
+def _build_order(description, levels, dependents, full_depends):
     levels = {}
     passed = set()
     all_keys = set(description['consts'].keys())
@@ -957,8 +994,9 @@ def assymbly(description):
                 description['order'][level].append(ready)
                 keys -= set(ready)
 
+
+def _make_axis(description):
     axis = {}
-    independent_variables = set(description['intrinsic_loops'].keys())
 
     for name in description['consts']:
         axis[name] = ()
@@ -967,8 +1005,6 @@ def assymbly(description):
             if isinstance(iterable, OptimizeSpace):
                 axis[name] = tuple(range(level + 1))
                 continue
-            elif isinstance(iterable, (np.ndarray, list, tuple, range, Space)):
-                independent_variables.add(name)
             axis[name] = (level, )
 
     for level, group in description['order'].items():
@@ -989,8 +1025,20 @@ def assymbly(description):
         k: tuple([x for x in v if x >= 0])
         for k, v in axis.items()
     }
+
+
+def assymbly(description):
+    _get_environment(description)
+    levels = _mapping_levels(description)
+    independent_variables = _get_independent_variables(description)
     description['independent_variables'] = independent_variables
 
+    dependents, full_depends, after_yield = _build_dependents(
+        description, levels, independent_variables)
+
+    _build_order(description, levels, dependents, full_depends)
+    _make_axis(description)
+
     return description
 
 

qulab/scan/server.py

@@ -20,6 +20,7 @@ from .models import Cell, Notebook
 from .models import Record as RecordInDB
 from .models import Session, create_engine, create_tables, sessionmaker, utcnow
 from .record import BufferList, Record, random_path
+from .utils import dump_dict, load_dict
 
 try:
     default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
@@ -53,6 +54,16 @@ class Request():
         return f"Request({self.method})"
 
 
+class Response():
+    pass
+
+
+class ErrorResponse(Response):
+
+    def __init__(self, error):
+        self.error = error
+
+
 async def reply(req, resp):
     await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
 
@@ -173,7 +184,7 @@ def record_delete(session: Session, record_id: int, datapath: Path):
     session.commit()
 
 
-@logger.catch
+@logger.catch(reraise=True)
 async def handle(session: Session, request: Request, datapath: Path):
 
     msg = request.msg
@@ -222,7 +233,7 @@ async def handle(session: Session, request: Request, datapath: Path):
             logger.debug(f"end bufferlist_iter_exit: {msg}")
         case 'record_create':
             logger.debug(f"record_create")
-            description = dill.loads(msg['description'])
+            description = load_dict(msg['description'])
             await reply(request, record_create(session, description, datapath))
             logger.debug(f"reply record_create")
         case 'record_append':
@@ -349,29 +360,16 @@ async def handle_with_timeout(session: Session, request: Request,
             f"Task handling request {request} timed out and was cancelled.")
         await reply(request, 'timeout')
     except Exception as e:
-        await reply(request, f'{e!r}')
+        logger.error(f"Task handling request {request} failed: {e!r}")
+        await reply(request, ErrorResponse(f'{e!r}'))
+    logger.debug(f"Task handling request {request} finished.")
 
 
 async def serv(port,
                datapath,
                url='',
                buffer_size=1024 * 1024 * 1024,
-               interval=60,
-               log='stderr',
-               debug=False):
-    if debug:
-        level = 'DEBUG'
-    else:
-        level = 'INFO'
-
-    if log == 'stderr':
-        pass
-        #logger.add(sys.stderr, level=level)
-    elif log == 'stdout':
-        logger.add(sys.stdout, level=level)
-    else:
-        logger.add(log, level=level)
-
+               interval=60):
     logger.debug('Creating socket...')
     async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
         logger.info(f'Server started at port {port}.')
@@ -392,8 +390,11 @@ async def serv(port,
                 received += len(msg)
                 try:
                     req = Request(sock, identity, msg)
-                except:
+                except Exception as e:
                     logger.exception('bad request')
+                    await sock.send_multipart(
+                        [identity,
+                         pickle.dumps(ErrorResponse(f'{e!r}'))])
                     continue
                 asyncio.create_task(
                     handle_with_timeout(session, req, datapath,
@@ -415,9 +416,21 @@ async def main(port,
                no_watch=True,
                debug=False):
     if no_watch:
+        logger.remove()
+        if debug:
+            level = 'DEBUG'
+        else:
+            level = 'INFO'
+        if log == 'stderr':
+            logger.add(sys.stderr, level=level)
+        elif log == 'stdout':
+            logger.add(sys.stdout, level=level)
+        else:
+            logger.add(sys.stderr, level=level)
+            logger.add(log, level=level)
+        logger.debug(f"logging level: {level}")
         logger.info('Server starting...')
-        await serv(port, datapath, url, buffer * 1024 * 1024, interval, log,
-                   debug)
+        await serv(port, datapath, url, buffer * 1024 * 1024, interval)
     else:
         process = None
 
@@ -444,10 +457,24 @@ async def main(port,
                         f'killed process. PID={process.pid}, returncode={process.returncode}'
                     )
                 cmd = [
-                    sys.executable, "-m", "qulab", "server", "--port",
-                    f"{port}", "--datapath", f"{datapath}", "--url", f"{url}",
-                    "--timeout", f"{timeout}", "--buffer", f"{buffer}",
-                    "--interval", f"{interval}", "--log", f"{log}",
+                    sys.executable,
+                    "-m",
+                    "qulab",
+                    "server",
+                    "--port",
+                    f"{port}",
+                    "--datapath",
+                    f"{datapath}",
+                    "--url",
+                    f"{url}",
+                    "--timeout",
+                    f"{timeout}",
+                    "--buffer",
+                    f"{buffer}",
+                    "--interval",
+                    f"{interval}",
+                    "--log",
+                    f"{log}",
                 ]
                 if url:
                     cmd.extend(['--url', url])
@@ -455,10 +482,7 @@ async def main(port,
                     cmd.append('--debug')
                 cmd.append("--no-watch")
                 logger.debug(f"starting process: {' '.join(cmd)}")
-                process = subprocess.Popen(cmd,
-                                           stdout=subprocess.PIPE,
-                                           stderr=subprocess.PIPE,
-                                           cwd=os.getcwd())
+                process = subprocess.Popen(cmd, cwd=os.getcwd())
                 logger.debug(
                     f'process started. PID={process.pid}, returncode={process.returncode}'
                 )
@@ -490,9 +514,15 @@ async def main(port,
 @click.option('--debug', is_flag=True, help='Debug mode.')
 def server(port, datapath, url, timeout, buffer, interval, log, no_watch,
            debug):
-    asyncio.run(
-        main(port, Path(datapath), url, timeout, buffer, interval, log,
-             True, debug))
+    try:
+        import uvloop
+        uvloop.run(
+            main(port, Path(datapath), url, timeout, buffer, interval, log,
+                 True, debug))
+    except ImportError:
+        asyncio.run(
+            main(port, Path(datapath), url, timeout, buffer, interval, log,
+                 True, debug))
 
 
 if __name__ == "__main__":

qulab/scan/utils.py

@@ -29,6 +29,46 @@ class TooLarge:
         return f'<TooLarge: {self.type} at 0x{id(self):x}>'
 
 
+def dump_dict(d, keys=[]):
+    ret = {}
+
+    for key, value in d.items():
+        if key in keys:
+            ret[key] = value
+            continue
+        if isinstance(value, dict) and isinstance(key, str):
+            ret[key] = dump_dict(value,
+                                 keys=[
+                                     k[len(key) + 1:] for k in keys
+                                     if k.startswith(f'{key}.')
+                                 ])
+        else:
+            try:
+                ret[key] = dill.dumps(value)
+            except:
+                ret[key] = Unpicklable(value)
+
+    return dill.dumps(ret)
+
+
+def load_dict(buff):
+    if isinstance(buff, dict):
+        return {key: load_dict(value) for key, value in buff.items()}
+
+    if not isinstance(buff, bytes):
+        return buff
+
+    try:
+        ret = dill.loads(buff)
+    except:
+        return buff
+
+    if isinstance(ret, dict):
+        return load_dict(ret)
+    else:
+        return ret
+
+
 def dump_globals(ns=None, *, size_limit=10 * 1024 * 1024, warn=False):
     import __main__
 

qulab/version.py

@@ -1 +1 @@
-__version__ = "2.3.3"
+__version__ = "2.3.4"