QuLab 2.0.3__cp311-cp311-win_amd64.whl → 2.0.5__cp311-cp311-win_amd64.whl

{QuLab-2.0.3.dist-info → QuLab-2.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.0.3
+Version: 2.0.5
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{QuLab-2.0.3.dist-info → QuLab-2.0.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=8zLGg-DfQhnDl2Ky0n-zXpN-8e-g7iR0AcaI4l4Vvpk,32
-qulab/__main__.py,sha256=OlYQe7go_eFLKR0uU6D1kIpTKvbpx3WRJFFdoYoFjdE,456
-qulab/fun.cp311-win_amd64.pyd,sha256=mPXp36Aw_1_IQZH_GaOYUJK9JZElr_Uy3bKRPiJ46Wg,31232
-qulab/version.py,sha256=HFL2NgNf74s56viRPRlgp_rAmKP1MzrtnJeQ8LxF_8M,21
+qulab/__main__.py,sha256=XN2wrhlmEkTIPq_ZeSaO8rWXfYgD2Czkm9DVFVoCw_U,515
+qulab/fun.cp311-win_amd64.pyd,sha256=ay6Srz8qiZP0Hsxatsz1d-2YK0FTjJ0Nsyp9TEiduMA,31232
+qulab/version.py,sha256=W2bKmLHMwuaZj0IuSoqothHJumPaUDyIwYJTzE6Hdd0,21
 qulab/monitor/__init__.py,sha256=xEVDkJF8issrsDeLqQmDsvtRmrf-UiViFcGTWuzdlFU,43
 qulab/monitor/__main__.py,sha256=k2H1H5Zf9LLXTDLISJkbikLH-z0f1e5i5i6wXXYPOrE,105
 qulab/monitor/config.py,sha256=y_5StMkdrbZO1ziyKBrvIkB7Jclp9RCPK1QbsOhCxnY,785
@@ -17,9 +17,10 @@ qulab/scan/curd.py,sha256=bEXtcmiaoAv5APToXx5O5tvmqAhE2_LkvdwLsI_8D5M,4662
 qulab/scan/expression.py,sha256=vwUM9E0OFQal4bljlUtLR3NJu4zGRyuWYrdyZSs3QTU,16199
 qulab/scan/models.py,sha256=TkiVHF_fUZzYHs4MsCTRh391thpf4Ozd3R_LAU0Gxkg,17657
 qulab/scan/optimize.py,sha256=MlT4y422CnP961IR384UKryyZh8riNvrPSd2z_MXLEg,2356
-qulab/scan/query_record.py,sha256=gRUBjablvjWhDM-nZEunMVxQbUeaFF8qG7sROy9kIbs,11882
-qulab/scan/recorder.py,sha256=EgS7NzARFqZeWUJp188iFQWr-_fa7Akm0Op-tvGqh7A,15849
-qulab/scan/scan.py,sha256=jDsjyOPf15DV8rk1q2CYH7aDO6ypaP6n7t3LmNaY03o,23187
+qulab/scan/query_record.py,sha256=rpw4U3NjLzlv9QMwKdCvEUGHjzPF8u1UpodfLW8aoTY,11853
+qulab/scan/recorder.py,sha256=wv8o_teAYYM_RaRQHkfa4-cF-ak68tzcb_QH9jlTH7A,18456
+qulab/scan/scan.py,sha256=nvvkGWmKWueeJ1pRAax3yKZn-vqlMvt10_oPSWd2hJw,26742
+qulab/scan/server.py,sha256=zDZfG6bOB3EUubfByQMq0BSQ9C6IV_Av0tDinzgpGjQ,2950
 qulab/scan/utils.py,sha256=XM-eKL5Xkm0hihhGS7Kq4g654Ye7n7TcU_f95gxtXq8,2634
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=6-EjN0waX1yfcMPJXqpIr9UlrIEsSCFApm5G-ZeaPMQ,1742
@@ -76,9 +77,9 @@ qulab/visualization/plot_layout.py,sha256=yAnMONOms7_szCdng-8wPpUMPis5UnbaNNzV4K
 qulab/visualization/plot_seq.py,sha256=h9D0Yl_yO64IwlvBgzMu9EBKr9gg6y8QE55gu2PfTns,2783
 qulab/visualization/qdat.py,sha256=HubXFu4nfcA7iUzghJGle1C86G6221hicLR0b-GqhKQ,5887
 qulab/visualization/widgets.py,sha256=HcYwdhDtLreJiYaZuN3LfofjJmZcLwjMfP5aasebgDo,3266
-QuLab-2.0.3.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
-QuLab-2.0.3.dist-info/METADATA,sha256=CBLJJtH8PMO1np50hxS8-Xbs1Bj4lFN1T7AwYCFI47Q,3609
-QuLab-2.0.3.dist-info/WHEEL,sha256=nSybvzWlmdJnHiUQSY-d7V1ycwEVUTqXiTvr2eshg44,102
-QuLab-2.0.3.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.0.3.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.0.3.dist-info/RECORD,,
+QuLab-2.0.5.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
+QuLab-2.0.5.dist-info/METADATA,sha256=siWaIXTJ0sU7pu5cBNaFGe2ecL8-tlbywjkxHYzGF-0,3609
+QuLab-2.0.5.dist-info/WHEEL,sha256=nSybvzWlmdJnHiUQSY-d7V1ycwEVUTqXiTvr2eshg44,102
+QuLab-2.0.5.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.0.5.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.0.5.dist-info/RECORD,,

qulab/__main__.py

@@ -2,6 +2,7 @@ import click
 
 from .monitor.__main__ import main as monitor
 from .scan.recorder import record
+from .scan.server import server
 from .sys.net.cli import dht
 from .visualization.__main__ import plot
 
@@ -21,6 +22,7 @@ main.add_command(monitor)
 main.add_command(plot)
 main.add_command(dht)
 main.add_command(record)
+main.add_command(server)
 
 if __name__ == '__main__':
     main()

qulab/scan/query_record.py

@@ -20,7 +20,6 @@ def get_record(id, database='tcp://127.0.0.1:6789'):
                 'record_id': id
             })
             d = dill.loads(socket.recv_pyobj())
-            print(d.keys())
             return Record(id, database, d)
     else:
         from .models import Record as RecordInDB

qulab/scan/recorder.py

@@ -1,9 +1,12 @@
 import asyncio
+import os
 import pickle
 import sys
 import time
 import uuid
+from collections import defaultdict
 from pathlib import Path
+from threading import Lock
 
 import click
 import dill
@@ -18,7 +21,16 @@ from .models import Record as RecordInDB
 from .models import Session, create_engine, create_tables, sessionmaker, utcnow
 
 _notgiven = object()
-datapath = Path.home() / 'qulab' / 'data'
+
+try:
+    default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
+except:
+    default_record_port = 6789
+
+if os.getenv('QULAB_RECORD_PATH'):
+    datapath = Path(os.getenv('QULAB_RECORD_PATH'))
+else:
+    datapath = Path.home() / 'qulab' / 'data'
 datapath.mkdir(parents=True, exist_ok=True)
 
 record_cache = {}
@@ -41,24 +53,49 @@ class BufferList():
         self.rd = ()
         self.pos_file = pos_file
         self.value_file = value_file
+        self._lock = Lock()
+
+    def __getstate__(self):
+        return {
+            'pos_file': self.pos_file,
+            'value_file': self.value_file,
+            '_pos': self._pos,
+            '_value': self._value,
+            'lu': self.lu,
+            'rd': self.rd
+        }
+
+    def __setstate__(self, state):
+        self.pos_file = state['pos_file']
+        self.value_file = state['value_file']
+        self._pos = state['_pos']
+        self._value = state['_value']
+        self.lu = state['lu']
+        self.rd = state['rd']
+        self._lock = Lock()
 
     @property
     def shape(self):
         return tuple([i - j for i, j in zip(self.rd, self.lu)])
 
     def flush(self):
-        if self.pos_file is not None:
-            with open(self.pos_file, 'ab') as f:
-                for pos in self._pos:
-                    dill.dump(pos, f)
-            self._pos.clear()
-        if self.value_file is not None:
-            with open(self.value_file, 'ab') as f:
-                for value in self._value:
-                    dill.dump(value, f)
-            self._value.clear()
-
-    def append(self, pos, value):
+        with self._lock:
+            if self.pos_file is not None:
+                with open(self.pos_file, 'ab') as f:
+                    for pos in self._pos:
+                        dill.dump(pos, f)
+                self._pos.clear()
+            if self.value_file is not None:
+                with open(self.value_file, 'ab') as f:
+                    for value in self._value:
+                        dill.dump(value, f)
+                self._value.clear()
+
+    def append(self, pos, value, dims=None):
+        if dims is not None:
+            if any([p != 0 for i, p in enumerate(pos) if i not in dims]):
+                return
+            pos = tuple([pos[i] for i in dims])
         self.lu = tuple([min(i, j) for i, j in zip(pos, self.lu)])
         self.rd = tuple([max(i + 1, j) for i, j in zip(pos, self.rd)])
         self._pos.append(pos)
@@ -68,25 +105,27 @@ class BufferList():
 
     def value(self):
         v = []
-        if self.value_file is not None:
-            with open(self.value_file, 'rb') as f:
-                while True:
-                    try:
-                        v.append(dill.load(f))
-                    except EOFError:
-                        break
+        if self.value_file is not None and self.value_file.exists():
+            with self._lock:
+                with open(self.value_file, 'rb') as f:
+                    while True:
+                        try:
+                            v.append(dill.load(f))
+                        except EOFError:
+                            break
         v.extend(self._value)
         return v
 
     def pos(self):
         p = []
-        if self.pos_file is not None:
-            with open(self.pos_file, 'rb') as f:
-                while True:
-                    try:
-                        p.append(dill.load(f))
-                    except EOFError:
-                        break
+        if self.pos_file is not None and self.pos_file.exists():
+            with self._lock:
+                with open(self.pos_file, 'rb') as f:
+                    while True:
+                        try:
+                            p.append(dill.load(f))
+                        except EOFError:
+                            break
         p.extend(self._pos)
         return p
 
@@ -114,21 +153,32 @@ class Record():
         self._file = None
         self.independent_variables = {}
         self.constants = {}
-
-        for level, group in self.description['order'].items():
-            for names in group:
-                for name in names:
-                    self._levels[name] = level
+        self.dims = {}
 
         for name, value in self.description['consts'].items():
             if name not in self._items:
                 self._items[name] = value
             self.constants[name] = value
+            self.dims[name] = ()
         for level, range_list in self.description['loops'].items():
             for name, iterable in range_list:
                 if isinstance(iterable, (np.ndarray, list, tuple, range)):
                     self._items[name] = iterable
-                    self.independent_variables[name] = (level, iterable)
+                    self.independent_variables[name] = iterable
+                    self.dims[name] = (level, )
+
+        for level, group in self.description['order'].items():
+            for names in group:
+                for name in names:
+                    self._levels[name] = level
+                    if name not in self.dims:
+                        if name not in self.description['dependents']:
+                            self.dims[name] = (level, )
+                        else:
+                            d = set()
+                            for n in self.description['dependents'][name]:
+                                d.update(self.dims[n])
+                            self.dims[name] = tuple(sorted(d))
 
         if self.is_local_record():
             self.database = Path(self.database)
@@ -203,6 +253,7 @@ class Record():
         for key in set(variables.keys()) - self._last_vars:
             if key not in self._levels:
                 self._levels[key] = level
+                self.dims[key] = tuple(range(level + 1))
 
         self._last_vars = set(variables.keys())
         self._keys.update(variables.keys())
@@ -237,9 +288,9 @@ class Record():
                         self._items[key] = BufferList()
                     self._items[key].lu = pos
                     self._items[key].rd = tuple([i + 1 for i in pos])
-                    self._items[key].append(pos, value)
+                    self._items[key].append(pos, value, self.dims[key])
                 elif isinstance(self._items[key], BufferList):
-                    self._items[key].append(pos, value)
+                    self._items[key].append(pos, value, self.dims[key])
             elif self._levels[key] == -1 and key not in self._items:
                 self._items[key] = value
 
@@ -254,6 +305,16 @@ class Record():
         with open(self._file, 'wb') as f:
             dill.dump(self, f)
 
+    def __repr__(self):
+        return f"<Record: id={self.id} app={self.description['app']}, keys={self.keys()}>"
+    
+    # def _repr_html_(self):
+    #     return f"""
+    #     <h3>Record: id={self.id}, app={self.description['app']}</h3>
+    #     <p>keys={self.keys()}</p>
+    #     <p>dims={self.dims}</p>
+    #     """
+
 
 class Request():
     __slots__ = ['sock', 'identity', 'msg', 'method']
@@ -283,7 +344,7 @@ def flush_cache():
         r.flush()
 
 
-def get_record(session, id, datapath):
+def get_record(session: Session, id: int, datapath: Path) -> Record:
     if id not in record_cache:
         record_in_db = session.get(RecordInDB, id)
         record_in_db.atime = utcnow()
@@ -297,7 +358,7 @@ def get_record(session, id, datapath):
     return record
 
 
-def create_record(session, description, datapath):
+def record_create(session: Session, description: dict, datapath: Path) -> int:
     record = Record(None, datapath, description)
     record_in_db = RecordInDB()
     if 'app' in description:
@@ -317,6 +378,20 @@ def create_record(session, description, datapath):
         raise
 
 
+def record_append(session: Session, record_id: int, level: int, step: int,
+                  position: int, variables: dict, datapath: Path):
+    record = get_record(session, record_id, datapath)
+    record.append(level, step, position, variables)
+    try:
+        record_in_db = session.get(RecordInDB, record_id)
+        record_in_db.mtime = utcnow()
+        record_in_db.atime = utcnow()
+        session.commit()
+    except:
+        session.rollback()
+        raise
+
+
 @logger.catch
 async def handle(session: Session, request: Request, datapath: Path):
 
@@ -327,16 +402,10 @@ async def handle(session: Session, request: Request, datapath: Path):
             await reply(request, 'pong')
         case 'record_create':
             description = dill.loads(msg['description'])
-            await reply(request, create_record(session, description, datapath))
+            await reply(request, record_create(session, description, datapath))
         case 'record_append':
-            record = get_record(session, msg['record_id'], datapath)
-            record.append(msg['level'], msg['step'], msg['position'],
-                          msg['variables'])
-            if msg['level'] < 0:
-                record_in_db = session.get(RecordInDB, msg['record_id'])
-                record_in_db.mtime = utcnow()
-                record_in_db.atime = utcnow()
-                session.commit()
+            record_append(session, msg['record_id'], msg['level'], msg['step'],
+                          msg['position'], msg['variables'], datapath)
         case 'record_description':
             record = get_record(session, msg['record_id'], datapath)
             await reply(request, dill.dumps(record.description))
@@ -365,7 +434,7 @@ async def handle(session: Session, request: Request, datapath: Path):
         case 'record_replace_tags':
             update_tags(session, msg['record_id'], msg['tags'], False)
         case _:
-            logger.error(f'Unknown method: {msg["method"]}')
+            logger.error(f"Unknown method: {msg['method']}")
 
 
 async def _handle(session: Session, request: Request, datapath: Path):
@@ -431,7 +500,9 @@ async def main(port, datapath, url, timeout=1, buffer=1024, interval=60):
 
 
 @click.command()
-@click.option('--port', default=6789, help='Port of the server.')
+@click.option('--port',
+              default=os.getenv('QULAB_RECORD_PORT', 6789),
+              help='Port of the server.')
 @click.option('--datapath', default=datapath, help='Path of the data.')
 @click.option('--url', default=None, help='URL of the database.')
 @click.option('--timeout', default=1, help='Timeout of ping.')

qulab/scan/scan.py

@@ -6,6 +6,7 @@ import os
 import re
 import sys
 import uuid
+import warnings
 from graphlib import TopologicalSorter
 from pathlib import Path
 from types import MethodType
@@ -21,7 +22,7 @@ from tqdm.notebook import tqdm
 from ..sys.rpc.zmq_socket import ZMQContextManager
 from .expression import Env, Expression, Symbol
 from .optimize import NgOptimizer
-from .recorder import Record
+from .recorder import Record, default_record_port
 from .utils import async_zip, call_function
 
 __process_uuid = uuid.uuid1()
@@ -171,7 +172,9 @@ class Scan():
     def __init__(self,
                  app: str = 'task',
                  tags: tuple[str] = (),
-                 database: str | Path | None = 'tcp://127.0.0.1:6789',
+                 database: str | Path
+                 | None = f'tcp://127.0.0.1:{default_record_port}',
+                 dump_globals: bool = False,
                  mixin=None):
         self.id = task_uuid()
         self.record = None
@@ -183,36 +186,61 @@ class Scan():
             'consts': {},
             'functions': {},
             'optimizers': {},
+            'namespace': {} if dump_globals else None,
             'actions': {},
             'dependents': {},
             'order': {},
             'filters': {},
-            'total': {}
+            'total': {},
+            'database': database,
+            'hiden': ['self', r'^__.*', r'.*__$'],
+            'entry': {
+                'env': {},
+                'shell': '',
+                'cmds': []
+            },
         }
         self._current_level = 0
-        self.variables = {}
-        self._task = None
-        self.sock = None
-        self.database = database
+        self._variables = {}
+        self._main_task = None
+        self._sock = None
         self._sem = asyncio.Semaphore(100)
         self._bar: dict[int, tqdm] = {}
-        self._hide_patterns = [r'^__.*', r'.*__$']
-        self._hide_pattern_re = re.compile('|'.join(self._hide_patterns))
+        self._hide_pattern_re = re.compile('|'.join(self.description['hiden']))
         self._task_queue = asyncio.Queue()
+        self._task_pool = []
+        self._single_step = True
+
+    def __del__(self):
+        try:
+            self._main_task.cancel()
+        except:
+            pass
+        for task in self._task_pool:
+            try:
+                task.cancel()
+            except:
+                pass
 
     def __getstate__(self) -> dict:
         state = self.__dict__.copy()
         del state['record']
-        del state['sock']
-        del state['_task']
+        del state['_sock']
+        del state['_main_task']
+        del state['_bar']
+        del state['_task_queue']
+        del state['_task_pool']
         del state['_sem']
         return state
 
     def __setstate__(self, state: dict) -> None:
         self.__dict__.update(state)
         self.record = None
-        self.sock = None
-        self._task = None
+        self._sock = None
+        self._main_task = None
+        self._bar = {}
+        self._task_queue = asyncio.Queue()
+        self._task_pool = []
         self._sem = asyncio.Semaphore(100)
         for opt in self.description['optimizers'].values():
             opt.scanner = self
@@ -221,13 +249,17 @@ class Scan():
     def current_level(self):
         return self._current_level
 
+    @property
+    def variables(self) -> dict[str, Any]:
+        return self._variables
+
     async def emit(self, current_level, step, position, variables: dict[str,
                                                                         Any]):
         for key, value in list(variables.items()):
             if inspect.isawaitable(value) and not self.hiden(key):
                 variables[key] = await value
-        if self.sock is not None:
-            await self.sock.send_pyobj({
+        if self._sock is not None:
+            await self._sock.send_pyobj({
                 'task': self.id,
                 'method': 'record_append',
                 'record_id': self.record.id,
@@ -240,11 +272,14 @@ class Scan():
                 }
             })
         else:
-            self.record.append(current_level, step, position, variables)
+            self.record.append(current_level, step, position, {
+                k: v
+                for k, v in variables.items() if not self.hiden(k)
+            })
 
     def hide(self, name: str):
-        self._hide_patterns.append(re.compile(name))
-        self._hide_pattern_re = re.compile('|'.join(self._hide_patterns))
+        self.description['hiden'].append(name)
+        self._hide_pattern_re = re.compile('|'.join(self.description['hiden']))
 
     def hiden(self, name: str) -> bool:
         return bool(self._hide_pattern_re.match(name))
@@ -260,24 +295,8 @@ class Scan():
             return True
 
     async def create_record(self):
-        import __main__
-        from IPython import get_ipython
-
-        ipy = get_ipython()
-        if ipy is not None:
-            scripts = ('ipython', ipy.user_ns['In'])
-        else:
-            try:
-                scripts = ('shell',
-                           [sys.executable, __main__.__file__, *sys.argv[1:]])
-            except:
-                scripts = ('', [])
-
-        self.description['ctime'] = datetime.datetime.now()
-        self.description['scripts'] = scripts
-        self.description['env'] = {k: v for k, v in os.environ.items()}
-        if self.sock is not None:
-            await self.sock.send_pyobj({
+        if self._sock is not None:
+            await self._sock.send_pyobj({
                 'task':
                 self.id,
                 'method':
@@ -286,9 +305,10 @@ class Scan():
                 dill.dumps(self.description)
             })
 
-            record_id = await self.sock.recv_pyobj()
-            return Record(record_id, self.database, self.description)
-        return Record(None, self.database, self.description)
+            record_id = await self._sock.recv_pyobj()
+            return Record(record_id, self.description['database'],
+                          self.description)
+        return Record(None, self.description['database'], self.description)
 
     def get(self, name: str):
         if name in self.description['consts']:
@@ -323,6 +343,10 @@ class Scan():
         self.description['filters'][level].append(func)
 
     def set(self, name: str, value):
+        try:
+            dill.dumps(value)
+        except:
+            raise ValueError('value is not serializable.')
         if isinstance(value, Expression):
             self.add_depends(name, value.symbols())
             self.description['functions'][name] = value
@@ -392,7 +416,9 @@ class Scan():
     async def _run(self):
         assymbly(self.description)
         task = asyncio.create_task(self._update_progress())
-        self.variables = self.description['consts'].copy()
+        self._task_pool.append(task)
+        self._variables = {'self': self}
+        self._variables.update(self.description['consts'])
         for level, total in self.description['total'].items():
             if total == np.inf:
                 total = None
@@ -402,11 +428,13 @@ class Scan():
                 if name in self.description['functions']:
                     self.variables[name] = await call_function(
                         self.description['functions'][name], self.variables)
-        if isinstance(self.database,
-                      str) and self.database.startswith("tcp://"):
-            async with ZMQContextManager(zmq.DEALER,
-                                         connect=self.database) as socket:
-                self.sock = socket
+        if isinstance(
+                self.description['database'],
+                str) and self.description['database'].startswith("tcp://"):
+            async with ZMQContextManager(
+                    zmq.DEALER,
+                    connect=self.description['database']) as socket:
+                self._sock = socket
                 self.record = await self.create_record()
                 await self.work()
         else:
@@ -414,23 +442,49 @@ class Scan():
             await self.work()
         for level, bar in self._bar.items():
             bar.close()
+
+        while not self._task_queue.empty():
+            evt = self._task_queue.get_nowait()
+            if isinstance(evt, asyncio.Event):
+                evt.set()
+            elif inspect.isawaitable(evt):
+                await evt
         task.cancel()
+        if self._single_step:
+            await self.emit(0, 0, 0, self.variables.copy())
+            await self.emit(-1, 0, 0, {})
         return self.variables
 
     async def done(self):
-        if self._task is not None:
+        if self._main_task is not None:
             try:
-                await self._task
+                await self._main_task
             except asyncio.CancelledError:
                 pass
 
+    def finished(self):
+        return self._main_task.done()
+
     def start(self):
         import asyncio
-        self._task = asyncio.create_task(self._run())
+        self._main_task = asyncio.create_task(self._run())
+
+    async def submit(self, server='tcp://127.0.0.1:6788'):
+        assymbly(self.description)
+        async with ZMQContextManager(zmq.DEALER, connect=server) as socket:
+            await socket.send_pyobj({
+                'method': 'submit',
+                'description': dill.dumps(self.description)
+            })
+            self.id = await socket.recv_pyobj()
+            await socket.send_pyobj({'method': 'get_record_id', 'id': self.id})
+            record_id = await socket.recv_pyobj()
+            self.record = Record(record_id, self.description['database'],
+                                 self.description)
 
     def cancel(self):
-        if self._task is not None:
-            self._task.cancel()
+        if self._main_task is not None:
+            self._main_task.cancel()
 
     async def _reset_progress_bar(self, level):
         if level in self._bar:
@@ -445,7 +499,6 @@ class Scan():
             return
         step = 0
         position = 0
-        task = None
         self._task_queue.put_nowait(
             self._reset_progress_bar(self.current_level))
         async for variables in _iter_level(
@@ -456,7 +509,8 @@ class Scan():
             self._current_level += 1
             if await self._filter(variables, self.current_level - 1):
                 yield variables
-                task = asyncio.create_task(
+                self._single_step = False
+                asyncio.create_task(
                     self.emit(self.current_level - 1, step, position,
                               variables.copy()))
                 step += 1
@@ -464,8 +518,6 @@ class Scan():
             self._current_level -= 1
             self._task_queue.put_nowait(
                 self._update_progress_bar(self.current_level, 1))
-        if task is not None:
-            await task
         if self.current_level == 0:
             await self.emit(self.current_level - 1, 0, 0, {})
             for name, value in self.variables.items():
@@ -519,7 +571,73 @@ class Scan():
             return await awaitable
 
 
+class Unpicklable:
+
+    def __init__(self, obj):
+        self.type = str(type(obj))
+        self.id = id(obj)
+
+    def __repr__(self):
+        return f'<Unpicklable: {self.type} at 0x{id(self):x}>'
+
+
+class TooLarge:
+
+    def __init__(self, obj):
+        self.type = str(type(obj))
+        self.id = id(obj)
+
+    def __repr__(self):
+        return f'<TooLarge: {self.type} at 0x{id(self):x}>'
+
+
+def dump_globals(ns=None, *, size_limit=10 * 1024 * 1024, warn=False):
+    import __main__
+
+    if ns is None:
+        ns = __main__.__dict__
+
+    namespace = {}
+
+    for name, value in ns.items():
+        try:
+            buf = dill.dumps(value)
+        except:
+            namespace[name] = Unpicklable(value)
+            if warn:
+                warnings.warn(f'Unpicklable: {name} {type(value)}')
+        if len(buf) > size_limit:
+            namespace[name] = TooLarge(value)
+            if warn:
+                warnings.warn(f'TooLarge: {name} {type(value)}')
+        else:
+            namespace[name] = buf
+
+    return namespace
+
+
 def assymbly(description):
+    import __main__
+    from IPython import get_ipython
+
+    if isinstance(description['namespace'], dict):
+        description['namespace'] = dump_globals()
+
+    ipy = get_ipython()
+    if ipy is not None:
+        description['entry']['shell'] = 'ipython'
+        description['entry']['cmds'] = ipy.user_ns['In']
+    else:
+        try:
+            description['entry']['shell'] = 'shell'
+            description['entry']['cmds'] = [
+                sys.executable, __main__.__file__, *sys.argv[1:]
+            ]
+        except:
+            pass
+
+    description['entry']['env'] = {k: v for k, v in os.environ.items()}
+
     mapping = {
         label: level
         for level, label in enumerate(

qulab/scan/server.py

@@ -0,0 +1,106 @@
+import asyncio
+import pickle
+import sys
+import time
+import uuid
+from pathlib import Path
+from .scan import Scan
+import click
+import dill
+import numpy as np
+import zmq
+from loguru import logger
+
+from qulab.sys.rpc.zmq_socket import ZMQContextManager
+
+pool = {}
+
+class Request():
+    __slots__ = ['sock', 'identity', 'msg', 'method']
+
+    def __init__(self, sock, identity, msg):
+        self.sock = sock
+        self.identity = identity
+        self.msg = pickle.loads(msg)
+        self.method = self.msg.get('method', '')
+
+
+async def reply(req, resp):
+    await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
+
+
+@logger.catch
+async def handle(request: Request):
+
+    msg = request.msg
+
+    match request.method:
+        case 'ping':
+            await reply(request, 'pong')
+        case 'submit':
+            description = dill.loads(msg['description'])
+            task = Scan()
+            task.description = description
+            task.start()
+            pool[task.id] = task
+            await reply(request, task.id)
+        case 'get_record_id':
+            task = pool.get(msg['id'])
+            for _ in range(10):
+                if task.record:
+                    await reply(request, task.record.id)
+                    break
+                await asyncio.sleep(1)
+            else:
+                await reply(request, None)
+        case _:
+            logger.error(f"Unknown method: {msg['method']}")
+
+
+async def _handle(request: Request):
+    try:
+        await handle(request)
+    except:
+        await reply(request, 'error')
+
+
+async def serv(port):
+    logger.info('Server starting.')
+    async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
+        logger.info('Server started.')
+        while True:
+            identity, msg = await sock.recv_multipart()
+            req = Request(sock, identity, msg)
+            asyncio.create_task(_handle(req))
+
+
+async def watch(port, timeout=1):
+    with ZMQContextManager(zmq.DEALER,
+                           connect=f"tcp://127.0.0.1:{port}") as sock:
+        sock.setsockopt(zmq.LINGER, 0)
+        while True:
+            try:
+                sock.send_pyobj({"method": "ping"})
+                if sock.poll(int(1000 * timeout)):
+                    sock.recv()
+                else:
+                    raise asyncio.TimeoutError()
+            except (zmq.error.ZMQError, asyncio.TimeoutError):
+                return asyncio.create_task(serv(port))
+            await asyncio.sleep(timeout)
+
+
+async def main(port, timeout=1):
+    task = await watch(port=port, timeout=timeout)
+    await task
+
+
+@click.command()
+@click.option('--port', default=6788, help='Port of the server.')
+@click.option('--timeout', default=1, help='Timeout of ping.')
+def server(port, timeout):
+    asyncio.run(main(port, timeout))
+
+
+if __name__ == "__main__":
+    server()

qulab/version.py

@@ -1 +1 @@
-__version__ = "2.0.3"
+__version__ = "2.0.5"