QuLab 2.2.3__cp312-cp312-win_amd64.whl → 2.2.5__cp312-cp312-win_amd64.whl

{QuLab-2.2.3.dist-info → QuLab-2.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.2.3
+Version: 2.2.5
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{QuLab-2.2.3.dist-info → QuLab-2.2.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=vkFybY8YSsQilYdThPRD83-btPAR41sy_WCXiM-6mME,141
 qulab/__main__.py,sha256=V7iokU7awstgjCeiF_hoOdFyrqJwC_4QetiLe7cWvOQ,454
-qulab/fun.cp312-win_amd64.pyd,sha256=2t7RpwvU1lE3bd7l089Xj3XUJszO9Bz8ZeBlIfG4HfY,32256
-qulab/version.py,sha256=Nt4yGCnk6mAxHxAbWGL2m0dPsKn_yhp3_F9LBmkhnvI,21
+qulab/fun.cp312-win_amd64.pyd,sha256=2bsHNMUaHgDI2Hdtivzp8n6F8T-iO9HkaoTVVD6dwP0,32256
+qulab/version.py,sha256=zyH0HuxKprjHeDzaU3CDFspw9f0RqC0qcLriAPqWD_M,21
 qulab/monitor/__init__.py,sha256=xEVDkJF8issrsDeLqQmDsvtRmrf-UiViFcGTWuzdlFU,43
 qulab/monitor/__main__.py,sha256=k2H1H5Zf9LLXTDLISJkbikLH-z0f1e5i5i6wXXYPOrE,105
 qulab/monitor/config.py,sha256=y_5StMkdrbZO1ziyKBrvIkB7Jclp9RCPK1QbsOhCxnY,785
@@ -17,11 +17,11 @@ qulab/scan/curd.py,sha256=yaTglGiS6mlk0GqDHi_w8T02XGBMvDZtXSdML7zDywk,7117
 qulab/scan/expression.py,sha256=vwUM9E0OFQal4bljlUtLR3NJu4zGRyuWYrdyZSs3QTU,16199
 qulab/scan/models.py,sha256=ZvXkJEt5Yz3Sjx0JKzYka-q2Uo-w_iVzHgH8A6DbjF0,18236
 qulab/scan/optimize.py,sha256=MlT4y422CnP961IR384UKryyZh8riNvrPSd2z_MXLEg,2356
-qulab/scan/query.py,sha256=ejxaejMVwvYkSIGMPb95OU3DjMVWy3ID-Ea8mtsbxIc,11917
-qulab/scan/record.py,sha256=iCGkzIAFPYnT6idc--VloEVtyJPnHr35xt09lqtX674,20150
-qulab/scan/scan.py,sha256=fNpcL5KKSAGA1y6qteSxmH0Ii_I3IdDv5-ihhx1HUAs,35457
-qulab/scan/server.py,sha256=h5CPS1zKtBs0cKvEBdmT8_EQ_cxyX8VXGEjPlk9iD2I,12549
-qulab/scan/space.py,sha256=PBa7Z6KnS690x79WS2YU3PTA_s5K6MCJ6JmS93vs2s4,5381
+qulab/scan/query.py,sha256=z94Ofbii4HZ4ZyZ7nos9-y37bH8-Xg4mA3QsF_aWx-c,11907
+qulab/scan/record.py,sha256=crlNIm1ki6td2Z284D78M276aso7KhMnr4qjCCOj1Kk,21644
+qulab/scan/scan.py,sha256=CNX2Z6vWMax9SqBWWYLghkz6TwIaLNnKypeyS7UCTfA,35773
+qulab/scan/server.py,sha256=ZWTnlK_YMfPTCoq6GnGpsZzSFGyoO4qF7aEGW9wCwhM,14542
+qulab/scan/space.py,sha256=fWsY99HDauk89f5ygqnJYBoeNFGTUkAGFssrw-YMHok,5453
 qulab/scan/utils.py,sha256=30qnYvyFyctwcWxOCUpCNxXgGysA7xdIDzYbjwxGUzA,3744
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=6-EjN0waX1yfcMPJXqpIr9UlrIEsSCFApm5G-ZeaPMQ,1742
@@ -78,9 +78,9 @@ qulab/visualization/plot_layout.py,sha256=yAnMONOms7_szCdng-8wPpUMPis5UnbaNNzV4K
 qulab/visualization/plot_seq.py,sha256=h9D0Yl_yO64IwlvBgzMu9EBKr9gg6y8QE55gu2PfTns,2783
 qulab/visualization/qdat.py,sha256=HubXFu4nfcA7iUzghJGle1C86G6221hicLR0b-GqhKQ,5887
 qulab/visualization/widgets.py,sha256=HcYwdhDtLreJiYaZuN3LfofjJmZcLwjMfP5aasebgDo,3266
-QuLab-2.2.3.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
-QuLab-2.2.3.dist-info/METADATA,sha256=QjGRvgbe0XCIgNaV05AHTHgLWLvbd5kaJi9Ceo73Pgc,3609
-QuLab-2.2.3.dist-info/WHEEL,sha256=fZWyj_84lK0cA-ZNCsdwhbJl0OTrpWkxInEn424qrSs,102
-QuLab-2.2.3.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.2.3.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.2.3.dist-info/RECORD,,
+QuLab-2.2.5.dist-info/LICENSE,sha256=b4NRQ-GFVpJMT7RuExW3NwhfbrYsX7AcdB7Gudok-fs,1086
+QuLab-2.2.5.dist-info/METADATA,sha256=BspoYai-nQ5IQpf_AqvzYlKLZ3QZwUfDbw9Cz_73yME,3609
+QuLab-2.2.5.dist-info/WHEEL,sha256=fZWyj_84lK0cA-ZNCsdwhbJl0OTrpWkxInEn424qrSs,102
+QuLab-2.2.5.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.2.5.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.2.5.dist-info/RECORD,,

qulab/scan/query.py

@@ -24,7 +24,7 @@ def get_record(id, database=default_server) -> Record:
                 'record_id': id
             })
             d = dill.loads(socket.recv_pyobj())
-            if isinstance(d, None):
+            if d is None:
                 raise ValueError(f'No record with id {id}')
             d.id = id
             d.database = database

qulab/scan/record.py

@@ -115,7 +115,8 @@ class BufferList():
                 self.inner_shape = value.shape
             elif self.inner_shape != value.shape:
                 self.inner_shape = ()
-        self._list.append((pos, value))
+        with self._lock:
+            self._list.append((pos, value))
         if len(self._list) > 1000:
             self.flush()
 
@@ -156,14 +157,29 @@ class BufferList():
         else:
             server, record_id, key = self._data_id
             with ZMQContextManager(zmq.DEALER, connect=server) as socket:
-                socket.send_pyobj({
-                    'method': 'bufferlist_slice',
-                    'record_id': record_id,
-                    'key': key,
-                    'slice': self._slice
-                })
-                ret = socket.recv_pyobj()
-                yield from ret
+                iter_id = b''
+                start = 0
+                try:
+                    while True:
+                        socket.send_pyobj({
+                            'method': 'bufferlist_iter',
+                            'record_id': record_id,
+                            'iter_id': iter_id,
+                            'key': key,
+                            'slice': self._slice,
+                            'start': start
+                        })
+                        iter_id, lst, finished = socket.recv_pyobj()
+                        start += len(lst)
+                        yield from lst
+                        if finished:
+                            break
+                finally:
+                    if iter_id:
+                        socket.send_pyobj({
+                            'method': 'bufferlist_iter_exit',
+                            'iter_id': iter_id
+                        })
 
     def value(self):
         d = []
@@ -304,6 +320,14 @@ class Record():
         self._last_vars = set()
         self._file = None
 
+        for name, value in self.description['intrinsic_loops'].items():
+            self._items[name] = value
+
+        for level, range_list in description['loops'].items():
+            for name, iterable in range_list:
+                if name in self.description['independent_variables']:
+                    self._items[name] = iterable
+
         if self.is_local_record():
             self.database = Path(self.database)
             self._file = random_path(self.database / 'objects')
@@ -395,6 +419,11 @@ class Record():
                     else:
                         ret._data_id = self.database, self.id, key
                         return ret
+                elif isinstance(ret, Space):
+                    if buffer_to_array:
+                        return ret.toarray()
+                    else:
+                        return ret
                 else:
                     return ret
         else:
@@ -410,6 +439,11 @@ class Record():
                     return d.array()
                 else:
                     return d
+            elif isinstance(d, Space):
+                if buffer_to_array:
+                    return d.toarray()
+                else:
+                    return d
             else:
                 return d
 
@@ -516,6 +550,8 @@ class Record():
             with z.open('record.pkl', 'w') as f:
                 self.description['entry']['scripts'] = self.scripts()
                 dill.dump((self.description, items), f)
+            with z.open('config.pkl', 'w') as f:
+                f.write(dill.dumps(self.config()))
 
     def scripts(self, session=None):
         scripts = self.description['entry']['scripts']

qulab/scan/scan.py

@@ -225,6 +225,7 @@ class Scan():
                  dump_globals: bool = False,
                  max_workers: int = 4,
                  max_promise: int = 100,
+                 max_message: int = 1000,
                  mixin=None):
         self.id = task_uuid()
         self.record = None
@@ -265,11 +266,12 @@ class Scan():
         self._sem = asyncio.Semaphore(max_promise + 1)
         self._bar: dict[int, tqdm] = {}
         self._hide_pattern_re = re.compile('|'.join(self.description['hiden']))
-        self._msg_queue = asyncio.Queue()
+        self._msg_queue = asyncio.Queue(max_message)
         self._prm_queue = asyncio.Queue()
         self._single_step = True
         self._max_workers = max_workers
         self._max_promise = max_promise
+        self._max_message = max_message
         self._executors = ProcessPoolExecutor(max_workers=max_workers)
 
     def __del__(self):
@@ -297,7 +299,7 @@ class Scan():
         self._main_task = None
         self._bar = {}
         self._prm_queue = asyncio.Queue()
-        self._msg_queue = asyncio.Queue()
+        self._msg_queue = asyncio.Queue(self._max_message)
         self._sem = asyncio.Semaphore(self._max_promise + 1)
         self._executors = ProcessPoolExecutor(max_workers=self._max_workers)
         for opt in self.description['optimizers'].values():
@@ -344,8 +346,9 @@ class Scan():
                 for k, v in variables.items() if not self.hiden(k)
             })
 
-    def emit(self, current_level, step, position, variables: dict[str, Any]):
-        self._msg_queue.put_nowait(
+    async def emit(self, current_level, step, position, variables: dict[str,
+                                                                        Any]):
+        await self._msg_queue.put(
             self._emit(current_level, step, position, variables.copy()))
 
     def hide(self, name: str):
@@ -561,7 +564,7 @@ class Scan():
             await self._run()
 
     async def _run(self):
-        send_msg = asyncio.create_task(self._send_msg())
+        send_msg_task = asyncio.create_task(self._send_msg())
         update_progress_task = asyncio.create_task(self._update_progress())
 
         self._variables = {'self': self}
@@ -596,13 +599,13 @@ class Scan():
                 self.description['order'].get(-1, []),
                 self.description['getters'], self.variables))
 
-            self.emit(0, 0, 0, self.variables)
-            self.emit(-1, 0, 0, {})
+            await self.emit(0, 0, 0, self.variables)
+            await self.emit(-1, 0, 0, {})
 
         await self._prm_queue.join()
         update_progress_task.cancel()
         await self._msg_queue.join()
-        send_msg.cancel()
+        send_msg_task.cancel()
         return self.variables
 
     async def done(self):
@@ -623,11 +626,14 @@ class Scan():
         assymbly(self.description)
         async with ZMQContextManager(zmq.DEALER, connect=server) as socket:
             await socket.send_pyobj({
-                'method': 'submit',
+                'method': 'task_submit',
                 'description': dill.dumps(self.description)
             })
             self.id = await socket.recv_pyobj()
-            await socket.send_pyobj({'method': 'get_record_id', 'id': self.id})
+            await socket.send_pyobj({
+                'method': 'task_get_record_id',
+                'id': self.id
+            })
             record_id = await socket.recv_pyobj()
             self.record = Record(record_id, self.description['database'],
                                  self.description)
@@ -661,14 +667,15 @@ class Scan():
             if await self._filter(variables, self.current_level - 1):
                 yield variables
                 self._single_step = False
-                self.emit(self.current_level - 1, step, position, variables)
+                await self.emit(self.current_level - 1, step, position,
+                                variables)
                 step += 1
             position += 1
             self._current_level -= 1
             self._prm_queue.put_nowait(
                 self._update_progress_bar(self.current_level, 1))
         if self.current_level == 0:
-            self.emit(self.current_level - 1, 0, 0, {})
+            await self.emit(self.current_level - 1, 0, 0, {})
             for name, value in self.variables.items():
                 if inspect.isawaitable(value):
                     self.variables[name] = await value

qulab/scan/server.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 import pickle
 import time
+import uuid
 from pathlib import Path
 
 import click
@@ -29,7 +30,9 @@ else:
     datapath = Path.home() / 'qulab' / 'data'
 datapath.mkdir(parents=True, exist_ok=True)
 
+namespace = uuid.uuid4()
 record_cache = {}
+buffer_list_cache = {}
 CACHE_SIZE = 1024
 
 pool = {}
@@ -44,6 +47,9 @@ class Request():
         self.msg = pickle.loads(msg)
         self.method = self.msg.get('method', '')
 
+    def __repr__(self):
+        return f"Request({self.method})"
+
 
 async def reply(req, resp):
     await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
@@ -57,6 +63,11 @@ def clear_cache():
                          range(len(record_cache) - CACHE_SIZE)):
         del record_cache[k]
 
+    for k, (t,
+            _) in zip(sorted(buffer_list_cache.items(), key=lambda x: x[1][0]),
+                      range(len(buffer_list_cache) - CACHE_SIZE)):
+        del buffer_list_cache[k]
+
 
 def flush_cache():
     for k, (t, r) in record_cache.items():
@@ -142,12 +153,37 @@ async def handle(session: Session, request: Request, datapath: Path):
     match request.method:
         case 'ping':
             await reply(request, 'pong')
-        case 'bufferlist_slice':
-            record = get_record(session, msg['record_id'], datapath)
-            bufferlist = record.get(msg['key'],
-                                    buffer_to_array=False,
-                                    slice=msg['slice'])
-            await reply(request, list(bufferlist.iter()))
+        case 'bufferlist_iter':
+            if msg['iter_id'] in buffer_list_cache:
+                it = buffer_list_cache[msg['iter_id']][1]
+                iter_id = msg['iter_id']
+            else:
+                iter_id = uuid.uuid3(namespace, str(time.time_ns())).bytes
+                record = get_record(session, msg['record_id'], datapath)
+                bufferlist = record.get(msg['key'],
+                                        buffer_to_array=False,
+                                        slice=msg['slice'])
+                it = bufferlist.iter()
+                for _, _ in zip(range(msg['start']), it):
+                    pass
+            current_time = time.time()
+            ret, end = [], False
+            while time.time() - current_time < 0.02:
+                try:
+                    ret.append(next(it))
+                except StopIteration:
+                    end = True
+                    break
+            await reply(request, (iter_id, ret, end))
+            buffer_list_cache[iter_id] = time.time(), it
+            clear_cache()
+        case 'bufferlist_iter_exit':
+            try:
+                it = buffer_list_cache.pop(msg['iter_id'])[1]
+                it.throw(Exception)
+            except:
+                pass
+            clear_cache()
         case 'record_create':
             description = dill.loads(msg['description'])
             await reply(request, record_create(session, description, datapath))
@@ -220,7 +256,7 @@ async def handle(session: Session, request: Request, datapath: Path):
                                                    'objects').parts[-4:]))
             session.commit()
             await reply(request, config.id)
-        case 'submit':
+        case 'task_submit':
             from .scan import Scan
             finished = [(id, queried) for id, (task, queried) in pool.items()
                         if not isinstance(task, int) and task.finished()]
@@ -235,7 +271,7 @@ async def handle(session: Session, request: Request, datapath: Path):
             task.start()
             pool[task.id] = [task, False]
             await reply(request, task.id)
-        case 'get_record_id':
+        case 'task_get_record_id':
             task, queried = pool.get(msg['id'])
             if isinstance(task, int):
                 await reply(request, task)
@@ -249,15 +285,28 @@ async def handle(session: Session, request: Request, datapath: Path):
                     await asyncio.sleep(1)
                 else:
                     await reply(request, None)
+        case 'task_get_progress':
+            task, _ = pool.get(msg['id'])
+            if isinstance(task, int):
+                await reply(request, 1)
+            else:
+                await reply(request,
+                            [(bar.n, bar.total) for bar in task._bar.values()])
         case _:
             logger.error(f"Unknown method: {msg['method']}")
 
 
-async def _handle(session: Session, request: Request, datapath: Path):
+async def handle_with_timeout(session: Session, request: Request,
+                              datapath: Path, timeout: float):
     try:
-        await handle(session, request, datapath)
-    except:
-        await reply(request, 'error')
+        await asyncio.wait_for(handle(session, request, datapath),
+                               timeout=timeout)
+    except asyncio.TimeoutError:
+        logger.warning(
+            f"Task handling request {request} timed out and was cancelled.")
+        await reply(request, 'timeout')
+    except Exception as e:
+        await reply(request, f'{e!r}')
 
 
 async def serv(port,
@@ -280,7 +329,8 @@ async def serv(port,
                 identity, msg = await sock.recv_multipart()
                 received += len(msg)
                 req = Request(sock, identity, msg)
-                asyncio.create_task(_handle(session, req, datapath))
+                asyncio.create_task(
+                    handle_with_timeout(session, req, datapath, timeout=60.0))
                 if received > buffer_size or time.time(
                 ) - last_flush_time > interval:
                     flush_cache()

qulab/scan/space.py

@@ -28,6 +28,8 @@ class Space():
             return space
         if isinstance(space, (list, tuple)):
             array = np.array(space)
+        elif isinstance(space, np.ndarray):
+            array = space
         try:
             a = np.linspace(array[0], array[-1], len(array), dtype=array.dtype)
             if np.allclose(a, array):

qulab/version.py

@@ -1 +1 @@
-__version__ = "2.2.3"
+__version__ = "2.2.5"