QuLab 2.0.4__cp311-cp311-macosx_10_9_universal2.whl → 2.0.6__cp311-cp311-macosx_10_9_universal2.whl

{QuLab-2.0.4.dist-info → QuLab-2.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.0.4
+Version: 2.0.6
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{QuLab-2.0.4.dist-info → QuLab-2.0.6.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=8zLGg-DfQhnDl2Ky0n-zXpN-8e-g7iR0AcaI4l4Vvpk,32
 qulab/__main__.py,sha256=eupSsrNVfnTFRpjgrY_knPvZIs0-Dk577LaN7qB15hI,487
-qulab/fun.cpython-311-darwin.so,sha256=Ts0LNKfAATzvuQVYqCs5zehDzRYyYEXNHT824hdGaGA,159616
-qulab/version.py,sha256=vd1KwOWsA9E1XbsJopSk63EogXGb_qCx8XG7lzXJd8Y,21
+qulab/fun.cpython-311-darwin.so,sha256=5n2eKR4XMC2UAch4iGZs8bvMxen1dAwPdaanMSzZYC4,159616
+qulab/version.py,sha256=IoNlI2ogLAGBFo-lP1Eg_QnxTNCxj3o2OCiBaMyJrP8,21
 qulab/monitor/__init__.py,sha256=nTHelnDpxRS_fl_B38TsN0njgq8eVTEz9IAnN3NbDlM,42
 qulab/monitor/__main__.py,sha256=w3yUcqq195LzSnXTkQcuC1RSFRhy4oQ_PEBmucXguME,97
 qulab/monitor/config.py,sha256=fQ5JcsMApKc1UwANEnIvbDQZl8uYW0tle92SaYtX9lI,744
@@ -18,8 +18,8 @@ qulab/scan/expression.py,sha256=-aTYbjFQNI1mwOcoSBztqhKfGJpu_n4a1QnWro_xnTU,1569
 qulab/scan/models.py,sha256=S8Q9hC8nOzxyoNB10EYg-miDKqoNMnjyAECjD-TuORw,17117
 qulab/scan/optimize.py,sha256=vErjRTCtn2MwMF5Xyhs1P4gHF2IFHv_EqxsUvH_4y7k,2287
 qulab/scan/query_record.py,sha256=BVkNgv3yfbMXX_Kguq18fvowKOBmBiiTvUwj_CqpiF4,11493
-qulab/scan/recorder.py,sha256=3-9iqYIg11Ne3vJ2Q8uk9Hp5x6elKBLlfh5EwslkgBA,17611
-qulab/scan/scan.py,sha256=a07dEFJE2JIVRFuySIuINYYdKzIyjbTcnO-i9oaC6_8,25598
+qulab/scan/recorder.py,sha256=0GXayspGWs-qFs5PZz25aMwTrl9Y0P-me0QDNmBQTeQ,22163
+qulab/scan/scan.py,sha256=g-2o0zr_BU5Ky0DZ6ksbcxFMpQdNnRuv22WyphF2rjA,26033
 qulab/scan/server.py,sha256=W8z3vr0cqSSKWzIG6_b0d-lpBpDXGpHSwN6VJuv3w9U,2844
 qulab/scan/utils.py,sha256=n5yquKlz2QYMzciPgD9vkpBJVgzVzOqAlfvB4Qu6oOk,2551
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -77,9 +77,9 @@ qulab/visualization/plot_layout.py,sha256=clNw9QjE_kVNpIIx2Ob4YhAz2fucPGMuzkoIrO
 qulab/visualization/plot_seq.py,sha256=lphYF4VhkEdc_wWr1kFBwrx2yujkyFPFaJ3pjr61awI,2693
 qulab/visualization/qdat.py,sha256=ZeevBYWkzbww4xZnsjHhw7wRorJCBzbG0iEu-XQB4EA,5735
 qulab/visualization/widgets.py,sha256=6KkiTyQ8J-ei70LbPQZAK35wjktY47w2IveOa682ftA,3180
-QuLab-2.0.4.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
-QuLab-2.0.4.dist-info/METADATA,sha256=ITxP9PCKiTxbultRyfTgA92HFzX5jch7pWrHZSVyCgE,3510
-QuLab-2.0.4.dist-info/WHEEL,sha256=eupBwbXGAhwNAPJSvj5BiShZwdZO8jnQ5yHfv-9aUGw,115
-QuLab-2.0.4.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.0.4.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.0.4.dist-info/RECORD,,
+QuLab-2.0.6.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
+QuLab-2.0.6.dist-info/METADATA,sha256=HXuU9olYtxNzfvx5keAhwCay9lNv0Mzs-7LXK5AuT4s,3510
+QuLab-2.0.6.dist-info/WHEEL,sha256=eupBwbXGAhwNAPJSvj5BiShZwdZO8jnQ5yHfv-9aUGw,115
+QuLab-2.0.6.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.0.6.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.0.6.dist-info/RECORD,,

qulab/scan/recorder.py

@@ -1,4 +1,5 @@
 import asyncio
+import itertools
 import os
 import pickle
 import sys
@@ -7,6 +8,7 @@ import uuid
 from collections import defaultdict
 from pathlib import Path
 from threading import Lock
+from types import EllipsisType
 
 import click
 import dill
@@ -44,52 +46,74 @@ def random_path(base):
             return path
 
 
+def index_in_slice(slice_obj: slice | int, index: int):
+    if isinstance(slice_obj, int):
+        return slice_obj == index
+    start, stop, step = slice_obj.start, slice_obj.stop, slice_obj.step
+    if start is None:
+        start = 0
+    if step is None:
+        step = 1
+    if stop is None:
+        stop = sys.maxsize
+
+    if step > 0:
+        return start <= index < stop and (index - start) % step == 0
+    else:
+        return stop < index <= start and (index - start) % step == 0
+
+
 class BufferList():
 
-    def __init__(self, pos_file=None, value_file=None):
-        self._pos = []
-        self._value = []
+    def __init__(self, file=None, slice=None):
+        self._list = []
         self.lu = ()
         self.rd = ()
-        self.pos_file = pos_file
-        self.value_file = value_file
+        self.inner_shape = None
+        self.file = file
+        self._slice = slice
         self._lock = Lock()
+        self._database = None
+
+    def __repr__(self):
+        return f"<BufferList: lu={self.lu}, rd={self.rd}, slice={self._slice}>"
 
     def __getstate__(self):
+        self.flush()
+        if isinstance(self.file, Path):
+            file = '/'.join(self.file.parts[-4:])
+        else:
+            file = self.file
         return {
-            'pos_file': self.pos_file,
-            'value_file': self.value_file,
-            '_pos': self._pos,
-            '_value': self._value,
+            'file': file,
             'lu': self.lu,
-            'rd': self.rd
+            'rd': self.rd,
+            'inner_shape': self.inner_shape,
         }
 
     def __setstate__(self, state):
-        self.pos_file = state['pos_file']
-        self.value_file = state['value_file']
-        self._pos = state['_pos']
-        self._value = state['_value']
+        self.file = state['file']
         self.lu = state['lu']
         self.rd = state['rd']
+        self.inner_shape = state['inner_shape']
+        self._list = []
+        self._slice = None
         self._lock = Lock()
+        self._database = None
 
     @property
     def shape(self):
         return tuple([i - j for i, j in zip(self.rd, self.lu)])
 
     def flush(self):
-        with self._lock:
-            if self.pos_file is not None:
-                with open(self.pos_file, 'ab') as f:
-                    for pos in self._pos:
-                        dill.dump(pos, f)
-                self._pos.clear()
-            if self.value_file is not None:
-                with open(self.value_file, 'ab') as f:
-                    for value in self._value:
-                        dill.dump(value, f)
-                self._value.clear()
+        if not self._list:
+            return
+        if isinstance(self.file, Path):
+            with self._lock:
+                with open(self.file, 'ab') as f:
+                    for item in self._list:
+                        dill.dump(item, f)
+                self._list.clear()
 
     def append(self, pos, value, dims=None):
         if dims is not None:
@@ -98,45 +122,96 @@ class BufferList():
             pos = tuple([pos[i] for i in dims])
         self.lu = tuple([min(i, j) for i, j in zip(pos, self.lu)])
         self.rd = tuple([max(i + 1, j) for i, j in zip(pos, self.rd)])
-        self._pos.append(pos)
-        self._value.append(value)
-        if len(self._value) > 1000:
+        if hasattr(value, 'shape'):
+            if self.inner_shape is None:
+                self.inner_shape = value.shape
+            elif self.inner_shape != value.shape:
+                self.inner_shape = ()
+        self._list.append((pos, value))
+        if len(self._list) > 1000:
             self.flush()
 
-    def value(self):
-        v = []
-        if self.value_file is not None and self.value_file.exists():
+    def _iter_file(self):
+        if isinstance(self.file, Path) and self.file.exists():
             with self._lock:
-                with open(self.value_file, 'rb') as f:
+                with open(self.file, 'rb') as f:
                     while True:
                         try:
-                            v.append(dill.load(f))
+                            pos, value = dill.load(f)
+                            yield pos, value
                         except EOFError:
                             break
-        v.extend(self._value)
-        return v
+
+    def iter(self):
+        for pos, value in itertools.chain(self._iter_file(), self._list):
+            if not self._slice:
+                yield pos, value
+            elif all([index_in_slice(s, i) for s, i in zip(self._slice, pos)]):
+                yield pos, value[self._slice[len(pos):]]
+
+    def value(self):
+        d = []
+        for pos, value in self.iter():
+            d.append(value)
+        return d
 
     def pos(self):
         p = []
-        if self.pos_file is not None and self.pos_file.exists():
-            with self._lock:
-                with open(self.pos_file, 'rb') as f:
-                    while True:
-                        try:
-                            p.append(dill.load(f))
-                        except EOFError:
-                            break
-        p.extend(self._pos)
+        for pos, value in self.iter():
+            p.append(pos)
         return p
 
+    def items(self):
+        p, d = [], []
+        for pos, value in self.iter():
+            p.append(pos)
+            d.append(value)
+        return p, d
+
     def array(self):
-        pos = np.asarray(self.pos()) - np.asarray(self.lu)
-        data = np.asarray(self.value())
+        pos, data = self.items()
+        pos = np.asarray(pos) - np.asarray(self.lu)
+        data = np.asarray(data)
         inner_shape = data.shape[1:]
         x = np.full(self.shape + inner_shape, np.nan, dtype=data[0].dtype)
         x.__setitem__(tuple(pos.T), data)
         return x
 
+    def _full_slice(self, slice_tuple: slice
+                    | tuple[slice | int | EllipsisType, ...]):
+        if isinstance(slice_tuple, slice):
+            slice_tuple = (slice_tuple, ) + (slice(0, sys.maxsize,
+                                                   1), ) * (len(self.lu) - 1)
+        if slice_tuple is Ellipsis:
+            slice_tuple = (slice(0, sys.maxsize, 1), ) * len(self.lu)
+        else:
+            head, tail = [], []
+            for i, s in enumerate(slice_tuple):
+                if s is Ellipsis:
+                    head = slice_tuple[:i]
+                    tail = slice_tuple[i + 1:]
+                    break
+            slice_tuple = head + (slice(0, sys.maxsize, 1), ) * (
+                len(self.lu) - len(head) - len(tail)) + tail
+        slice_list = []
+        for s in slice_tuple:
+            if isinstance(s, int):
+                slice_list.append(s)
+            else:
+                start, stop, step = s.start, s.stop, s.step
+                if start is None:
+                    start = 0
+                if step is None:
+                    step = 1
+                if stop is None:
+                    stop = sys.maxsize
+                slice_list.append(slice(start, stop, step))
+        return tuple(slice_list)
+
+    def __getitem__(self, slice_tuple: slice | EllipsisType
+                    | tuple[slice | int | EllipsisType, ...]):
+        return super().__getitem__(self._full_slice(slice_tuple))
+
 
 class Record():
 
@@ -149,7 +224,6 @@ class Record():
         self._index = []
         self._pos = []
         self._last_vars = set()
-        self._levels = {}
         self._file = None
         self.independent_variables = {}
         self.constants = {}
@@ -170,7 +244,6 @@ class Record():
         for level, group in self.description['order'].items():
             for names in group:
                 for name in names:
-                    self._levels[name] = level
                     if name not in self.dims:
                         if name not in self.description['dependents']:
                             self.dims[name] = (level, )
@@ -185,6 +258,35 @@ class Record():
             self._file = random_path(self.database / 'objects')
             self._file.parent.mkdir(parents=True, exist_ok=True)
 
+    def __getstate__(self) -> dict:
+        return {
+            'id': self.id,
+            'database': self.database,
+            'description': self.description,
+            '_keys': self._keys,
+            '_items': self._items,
+            '_index': self._index,
+            '_pos': self._pos,
+            '_last_vars': self._last_vars,
+            'independent_variables': self.independent_variables,
+            'constants': self.constants,
+            'dims': self.dims,
+        }
+
+    def __setstate__(self, state: dict):
+        self.id = state['id']
+        self.database = state['database']
+        self.description = state['description']
+        self._keys = state['_keys']
+        self._items = state['_items']
+        self._index = state['_index']
+        self._pos = state['_pos']
+        self._last_vars = state['_last_vars']
+        self.independent_variables = state['independent_variables']
+        self.constants = state['constants']
+        self.dims = state['dims']
+        self._file = None
+
     def is_local_record(self):
         return not self.is_cache_record() and not self.is_remote_record()
 
@@ -201,7 +303,7 @@ class Record():
     def __getitem__(self, key):
         return self.get(key)
 
-    def get(self, key, default=_notgiven, buffer_to_array=True):
+    def get(self, key, default=_notgiven, buffer_to_array=True, slice=None):
         if self.is_remote_record():
             with ZMQContextManager(zmq.DEALER,
                                    connect=self.database) as socket:
@@ -211,8 +313,21 @@ class Record():
                     'key': key
                 })
                 ret = socket.recv_pyobj()
-                if isinstance(ret, BufferList) and buffer_to_array:
-                    return ret.array()
+                if isinstance(ret, BufferList):
+                    socket.send_pyobj({
+                        'method': 'bufferlist_slice',
+                        'record_id': self.id,
+                        'key': key,
+                        'slice': slice
+                    })
+                    lst = socket.recv_pyobj()
+                    ret._list = lst
+                    ret._slice = slice
+                    if buffer_to_array:
+                        return ret.array()
+                    else:
+                        ret._database = self.database
+                        return ret
                 else:
                     return ret
         else:
@@ -221,15 +336,13 @@ class Record():
             else:
                 d = self._items.get(key, default)
             if isinstance(d, BufferList):
+                if isinstance(d.file, str):
+                    d.file = self._file.parent.parent.parent.parent / d.file
+                d._slice = slice
                 if buffer_to_array:
                     return d.array()
                 else:
-                    ret = BufferList()
-                    ret._pos = d.pos()
-                    ret._value = d.value()
-                    ret.lu = d.lu
-                    ret.rd = d.rd
-                    return ret
+                    return d
             else:
                 return d
 
@@ -251,8 +364,7 @@ class Record():
             return
 
         for key in set(variables.keys()) - self._last_vars:
-            if key not in self._levels:
-                self._levels[key] = level
+            if key not in self.dims:
                 self.dims[key] = tuple(range(level + 1))
 
         self._last_vars = set(variables.keys())
@@ -276,14 +388,17 @@ class Record():
             self._pos[-1] += 1
 
         for key, value in variables.items():
-            if level == self._levels[key]:
+            if self.dims[key] == ():
+                if key not in self._items:
+                    self._items[key] = value
+            elif level == self.dims[key][-1]:
                 if key not in self._items:
                     if self.is_local_record():
-                        f1 = random_path(self.database / 'objects')
-                        f1.parent.mkdir(parents=True, exist_ok=True)
-                        f2 = random_path(self.database / 'objects')
-                        f2.parent.mkdir(parents=True, exist_ok=True)
-                        self._items[key] = BufferList(f1, f2)
+                        bufferlist_file = random_path(self.database /
+                                                      'objects')
+                        bufferlist_file.parent.mkdir(parents=True,
+                                                     exist_ok=True)
+                        self._items[key] = BufferList(bufferlist_file)
                     else:
                         self._items[key] = BufferList()
                     self._items[key].lu = pos
@@ -291,8 +406,6 @@ class Record():
                     self._items[key].append(pos, value, self.dims[key])
                 elif isinstance(self._items[key], BufferList):
                     self._items[key].append(pos, value, self.dims[key])
-            elif self._levels[key] == -1 and key not in self._items:
-                self._items[key] = value
 
     def flush(self):
         if self.is_remote_record() or self.is_cache_record():
@@ -305,6 +418,16 @@ class Record():
         with open(self._file, 'wb') as f:
             dill.dump(self, f)
 
+    def __repr__(self):
+        return f"<Record: id={self.id} app={self.description['app']}, keys={self.keys()}>"
+
+    # def _repr_html_(self):
+    #     return f"""
+    #     <h3>Record: id={self.id}, app={self.description['app']}</h3>
+    #     <p>keys={self.keys()}</p>
+    #     <p>dims={self.dims}</p>
+    #     """
+
 
 class Request():
     __slots__ = ['sock', 'identity', 'msg', 'method']
@@ -341,6 +464,7 @@ def get_record(session: Session, id: int, datapath: Path) -> Record:
         path = datapath / 'objects' / record_in_db.file
         with open(path, 'rb') as f:
             record = dill.load(f)
+            record._file = path
     else:
         record = record_cache[id][1]
     clear_cache()
@@ -390,6 +514,12 @@ async def handle(session: Session, request: Request, datapath: Path):
     match request.method:
         case 'ping':
             await reply(request, 'pong')
+        case 'bufferlist_slice':
+            record = get_record(session, msg['record_id'], datapath)
+            bufferlist = record.get(msg['key'],
+                                    buffer_to_array=False,
+                                    slice=msg['slice'])
+            await reply(request, list(bufferlist.iter()))
         case 'record_create':
             description = dill.loads(msg['description'])
             await reply(request, record_create(session, description, datapath))

qulab/scan/scan.py

@@ -174,6 +174,7 @@ class Scan():
                  tags: tuple[str] = (),
                  database: str | Path
                  | None = f'tcp://127.0.0.1:{default_record_port}',
+                 dump_globals: bool = False,
                  mixin=None):
         self.id = task_uuid()
         self.record = None
@@ -185,7 +186,7 @@ class Scan():
             'consts': {},
             'functions': {},
             'optimizers': {},
-            'namespace': {},
+            'namespace': {} if dump_globals else None,
             'actions': {},
             'dependents': {},
             'order': {},
@@ -208,6 +209,7 @@ class Scan():
         self._hide_pattern_re = re.compile('|'.join(self.description['hiden']))
         self._task_queue = asyncio.Queue()
         self._task_pool = []
+        self._single_step = True
 
     def __del__(self):
         try:
@@ -448,6 +450,9 @@ class Scan():
             elif inspect.isawaitable(evt):
                 await evt
         task.cancel()
+        if self._single_step:
+            await self.emit(0, 0, 0, self.variables.copy())
+            await self.emit(-1, 0, 0, {})
         return self.variables
 
     async def done(self):
@@ -504,6 +509,7 @@ class Scan():
             self._current_level += 1
             if await self._filter(variables, self.current_level - 1):
                 yield variables
+                self._single_step = False
                 asyncio.create_task(
                     self.emit(self.current_level - 1, step, position,
                               variables.copy()))
@@ -555,10 +561,13 @@ class Scan():
         Returns:
             Promise: A promise object.
         """
-        async with self._sem:
-            task = asyncio.create_task(self._await(awaitable))
-            self._task_queue.put_nowait(task)
-            return Promise(task)
+        if inspect.isawaitable(awaitable):
+            async with self._sem:
+                task = asyncio.create_task(self._await(awaitable))
+                self._task_queue.put_nowait(task)
+                return Promise(task)
+        else:
+            return awaitable
 
     async def _await(self, awaitable: Awaitable):
         async with self._sem:
@@ -614,7 +623,8 @@ def assymbly(description):
     import __main__
     from IPython import get_ipython
 
-    description['namespace'] = dump_globals()
+    if isinstance(description['namespace'], dict):
+        description['namespace'] = dump_globals()
 
     ipy = get_ipython()
     if ipy is not None:

qulab/version.py

@@ -1 +1 @@
-__version__ = "2.0.4"
+__version__ = "2.0.6"