QuLab 2.0.9__cp310-cp310-macosx_10_9_universal2.whl → 2.1.1__cp310-cp310-macosx_10_9_universal2.whl

{QuLab-2.0.9.dist-info → QuLab-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.0.9
+Version: 2.1.1
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{QuLab-2.0.9.dist-info → QuLab-2.1.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=8zLGg-DfQhnDl2Ky0n-zXpN-8e-g7iR0AcaI4l4Vvpk,32
 qulab/__main__.py,sha256=eupSsrNVfnTFRpjgrY_knPvZIs0-Dk577LaN7qB15hI,487
-qulab/fun.cpython-310-darwin.so,sha256=W7qCKW1DOGziGD9ZuExLJIXamwySro5tF5ZYoHEhc_8,159632
-qulab/version.py,sha256=KmRUsw1os9zycj-k3YLWzgxp2vob6KBciPMbxED0Lqk,21
+qulab/fun.cpython-310-darwin.so,sha256=dRP29bQVmRqSAnZ0uefJfYrLLmCYvAtcgQYC_i3tQ3I,159632
+qulab/version.py,sha256=k4SipKIh6P-kJf4P7Os3KD55dGIg3hoiS5pK316IoMg,21
 qulab/monitor/__init__.py,sha256=nTHelnDpxRS_fl_B38TsN0njgq8eVTEz9IAnN3NbDlM,42
 qulab/monitor/__main__.py,sha256=w3yUcqq195LzSnXTkQcuC1RSFRhy4oQ_PEBmucXguME,97
 qulab/monitor/config.py,sha256=fQ5JcsMApKc1UwANEnIvbDQZl8uYW0tle92SaYtX9lI,744
@@ -12,16 +12,18 @@ qulab/monitor/monitor.py,sha256=7E4bnTsO6qC85fs2ONrccGHfaYKv7SW74mtXzv6QjVc,2305
 qulab/monitor/ploter.py,sha256=CbiIjmohgtwDDTVeGzhXEGVo3XjytMdhLwU9VUkg9vo,3601
 qulab/monitor/qt_compat.py,sha256=OK71_JSO_iyXjRIKHANmaK4Lx4bILUzmXI-mwKg3QeI,788
 qulab/monitor/toolbar.py,sha256=WEag6cxAtEsOLL14XvM7pSE56EA3MO188_JuprNjdBs,7948
-qulab/scan/__init__.py,sha256=BflmtNkyV72f4jqkeXoO_gi7kf8CHyk-AmKHjNXwe1A,124
+qulab/scan/__init__.py,sha256=9sDntupqDiSB1-V86unzO-UUSID28Q48meBr51bZKI8,117
 qulab/scan/curd.py,sha256=ntpK62ArZiF2mrDDewcw227VMR1E_8no0yLJSrgdgng,4518
 qulab/scan/expression.py,sha256=-aTYbjFQNI1mwOcoSBztqhKfGJpu_n4a1QnWro_xnTU,15694
 qulab/scan/models.py,sha256=S8Q9hC8nOzxyoNB10EYg-miDKqoNMnjyAECjD-TuORw,17117
 qulab/scan/optimize.py,sha256=vErjRTCtn2MwMF5Xyhs1P4gHF2IFHv_EqxsUvH_4y7k,2287
-qulab/scan/query_record.py,sha256=BVkNgv3yfbMXX_Kguq18fvowKOBmBiiTvUwj_CqpiF4,11493
-qulab/scan/recorder.py,sha256=uovxS752kCAJYK3PF3abP57kyZyVUS2jWBTVgs4zPzM,22532
-qulab/scan/scan.py,sha256=Te5PgJDAdbgaGVXOLOdTMzuGGUp-VzPW_9_z9hj52lQ,28438
-qulab/scan/server.py,sha256=W8z3vr0cqSSKWzIG6_b0d-lpBpDXGpHSwN6VJuv3w9U,2844
-qulab/scan/utils.py,sha256=n5yquKlz2QYMzciPgD9vkpBJVgzVzOqAlfvB4Qu6oOk,2551
+qulab/scan/query.py,sha256=RyCdYpyNiW1_NI5l7BdQLcxrNv9oNJglYrjikXjnxk8,11579
+qulab/scan/record.py,sha256=rZM8fJo2wtQbczWi9a8bFY_vAKbaqcpDm2dviaSfKUc,15087
+qulab/scan/recorder.py,sha256=Dy1kkXAq6jb8RuVH73lSmYb-S6Ob8BJH9HLf4heuDEs,8563
+qulab/scan/scan.py,sha256=0FhBYlV9y6ZFHXoiiB1nTKKHUiI0C6hbek1Gg4LolTc,29143
+qulab/scan/server.py,sha256=zpUbYcRSC3UI9z7yiIBnvzKR9vWejJTTJ7AwEt5iyJA,2768
+qulab/scan/space.py,sha256=nwSGGnppe-Z6WPHfOqX4eeD5AutMQlIiPwrkxBZuI9I,5209
+qulab/scan/utils.py,sha256=Pg_tCf3SUKTiPSBqb6Enkgx4bAyQJAkDGe9uYys1xVU,3613
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=3emxxRry8BB0m8hUZvJ_oBqkPy7ksV7flHB_KEDXZuI,1692
 qulab/storage/base_dataset.py,sha256=4aKhqBNdZfdlm_z1Qy5dv0HrvgpvMdy8pbyfua8uE-4,11865
@@ -77,9 +79,9 @@ qulab/visualization/plot_layout.py,sha256=clNw9QjE_kVNpIIx2Ob4YhAz2fucPGMuzkoIrO
 qulab/visualization/plot_seq.py,sha256=lphYF4VhkEdc_wWr1kFBwrx2yujkyFPFaJ3pjr61awI,2693
 qulab/visualization/qdat.py,sha256=ZeevBYWkzbww4xZnsjHhw7wRorJCBzbG0iEu-XQB4EA,5735
 qulab/visualization/widgets.py,sha256=6KkiTyQ8J-ei70LbPQZAK35wjktY47w2IveOa682ftA,3180
-QuLab-2.0.9.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
-QuLab-2.0.9.dist-info/METADATA,sha256=NeeeZDjT_WmdUgOo03cVKLo23y1BOjfhB4GO_eji-UY,3510
-QuLab-2.0.9.dist-info/WHEEL,sha256=r7U64H7df5k5VoE41bE2otJ6YmrMps4wUd2_S2hHvHQ,115
-QuLab-2.0.9.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.0.9.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.0.9.dist-info/RECORD,,
+QuLab-2.1.1.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
+QuLab-2.1.1.dist-info/METADATA,sha256=dQEG-sBYuWYurQrpv5oNSZED6N7c0eOqSx5hXHV8w4A,3510
+QuLab-2.1.1.dist-info/WHEEL,sha256=r7U64H7df5k5VoE41bE2otJ6YmrMps4wUd2_S2hHvHQ,115
+QuLab-2.1.1.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.1.1.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.1.1.dist-info/RECORD,,

qulab/scan/__init__.py

@@ -1,3 +1,3 @@
 from .expression import Expression, Symbol
-from .query_record import get_record, lookup, lookup_list
+from .query import get_record, lookup, lookup_list
 from .scan import Scan

qulab/scan/{query_record.py → query.py}

@@ -9,10 +9,11 @@ from IPython.display import display
 
 from qulab.sys.rpc.zmq_socket import ZMQContextManager
 
-from .recorder import Record
+from .record import Record
+from .scan import default_server
 
 
-def get_record(id, database='tcp://127.0.0.1:6789'):
+def get_record(id, database=default_server) -> Record:
     if isinstance(database, str) and database.startswith('tcp://'):
         with ZMQContextManager(zmq.DEALER, connect=database) as socket:
             socket.send_pyobj({
@@ -20,7 +21,10 @@ def get_record(id, database='tcp://127.0.0.1:6789'):
                 'record_id': id
             })
             d = dill.loads(socket.recv_pyobj())
-            return Record(id, database, d)
+            d.id = id
+            d.database = database
+            d._file = None
+            return d
     else:
         from .models import Record as RecordInDB
         from .models import create_engine, sessionmaker
@@ -292,7 +296,7 @@ def _on_tags_submit(tags, ui_widgets):
     _update_view(ui_widgets)
 
 
-def lookup(app=None, limit=10, database='tcp://127.0.0.1:6789'):
+def lookup(app=None, limit=10, database=default_server):
     after = widgets.DatePicker()
     before = widgets.DatePicker()
     app_prefix = widgets.Label('App:')

qulab/scan/record.py

@@ -0,0 +1,448 @@
+import itertools
+import sys
+import uuid
+from pathlib import Path
+from threading import Lock
+from types import EllipsisType
+
+import dill
+import numpy as np
+import zmq
+
+from qulab.sys.rpc.zmq_socket import ZMQContextManager
+
+from .space import OptimizeSpace, Space
+
+_notgiven = object()
+
+
+def random_path(base):
+    while True:
+        s = uuid.uuid4().hex
+        path = base / s[:2] / s[2:4] / s[4:6] / s[6:]
+        if not path.exists():
+            return path
+
+
+def index_in_slice(slice_obj: slice | int, index: int):
+    if isinstance(slice_obj, int):
+        return slice_obj == index
+    start, stop, step = slice_obj.start, slice_obj.stop, slice_obj.step
+    if start is None:
+        start = 0
+    if step is None:
+        step = 1
+    if stop is None:
+        stop = sys.maxsize
+
+    if step > 0:
+        return start <= index < stop and (index - start) % step == 0
+    else:
+        return stop < index <= start and (index - start) % step == 0
+
+
+class BufferList():
+
+    def __init__(self, file=None, slice=None):
+        self._list = []
+        self.lu = ()
+        self.rd = ()
+        self.inner_shape = ()
+        self.file = file
+        self._slice = slice
+        self._lock = Lock()
+        self._data_id = None
+
+    def __repr__(self):
+        return f"<BufferList: shape={self.shape}, lu={self.lu}, rd={self.rd}, slice={self._slice}>"
+
+    def __getstate__(self):
+        self.flush()
+        if isinstance(self.file, Path):
+            file = '/'.join(self.file.parts[-4:])
+        else:
+            file = self.file
+        return {
+            'file': file,
+            'lu': self.lu,
+            'rd': self.rd,
+            'inner_shape': self.inner_shape,
+        }
+
+    def __setstate__(self, state):
+        self.file = state['file']
+        self.lu = state['lu']
+        self.rd = state['rd']
+        self.inner_shape = state['inner_shape']
+        self._list = []
+        self._slice = None
+        self._lock = Lock()
+        self._data_id = None
+
+    @property
+    def shape(self):
+        return tuple([i - j
+                      for i, j in zip(self.rd, self.lu)]) + self.inner_shape
+
+    def flush(self):
+        if not self._list:
+            return
+        if isinstance(self.file, Path):
+            with self._lock:
+                with open(self.file, 'ab') as f:
+                    for item in self._list:
+                        dill.dump(item, f)
+                self._list.clear()
+
+    def append(self, pos, value, dims=None):
+        if dims is not None:
+            if any([p != 0 for i, p in enumerate(pos) if i not in dims]):
+                return
+            pos = tuple([pos[i] for i in dims])
+        self.lu = tuple([min(i, j) for i, j in zip(pos, self.lu)])
+        self.rd = tuple([max(i + 1, j) for i, j in zip(pos, self.rd)])
+        if hasattr(value, 'shape'):
+            if self.inner_shape is None:
+                self.inner_shape = value.shape
+            elif self.inner_shape != value.shape:
+                self.inner_shape = ()
+        self._list.append((pos, value))
+        if len(self._list) > 1000:
+            self.flush()
+
+    def _iter_file(self):
+        if isinstance(self.file, Path) and self.file.exists():
+            with self._lock:
+                with open(self.file, 'rb') as f:
+                    while True:
+                        try:
+                            pos, value = dill.load(f)
+                            yield pos, value
+                        except EOFError:
+                            break
+
+    def iter(self):
+        if self._data_id is None:
+            for pos, value in itertools.chain(self._iter_file(), self._list):
+                if not self._slice:
+                    yield pos, value
+                elif all(
+                    [index_in_slice(s, i) for s, i in zip(self._slice, pos)]):
+                    if self.inner_shape:
+                        yield pos, value[self._slice[len(pos):]]
+                    else:
+                        yield pos, value
+        else:
+            server, record_id, key = self._data_id
+            with ZMQContextManager(zmq.DEALER, connect=server) as socket:
+                socket.send_pyobj({
+                    'method': 'bufferlist_slice',
+                    'record_id': record_id,
+                    'key': key,
+                    'slice': self._slice
+                })
+                ret = socket.recv_pyobj()
+                yield from ret
+
+    def value(self):
+        d = []
+        for pos, value in self.iter():
+            d.append(value)
+        return d
+
+    def pos(self):
+        p = []
+        for pos, value in self.iter():
+            p.append(pos)
+        return p
+
+    def items(self):
+        p, d = [], []
+        for pos, value in self.iter():
+            p.append(pos)
+            d.append(value)
+        return p, d
+
+    def array(self):
+        pos, data = self.items()
+        if self._slice:
+            pos = np.asarray(pos)
+            lu = tuple(np.min(pos, axis=0))
+            rd = tuple(np.max(pos, axis=0) + 1)
+            pos = np.asarray(pos) - np.asarray(lu)
+            shape = []
+            for k, (s, i, j) in enumerate(zip(self._slice, rd, lu)):
+                if s.step is not None:
+                    pos[:, k] = pos[:, k] / s.step
+                    shape.append(round(np.ceil((i - j) / s.step)))
+                else:
+                    shape.append(i - j)
+            shape = tuple(shape)
+        else:
+            shape = tuple([i - j for i, j in zip(self.rd, self.lu)])
+            pos = np.asarray(pos) - np.asarray(self.lu)
+        data = np.asarray(data)
+        inner_shape = data.shape[1:]
+        x = np.full(shape + inner_shape, np.nan, dtype=data[0].dtype)
+        x.__setitem__(tuple(pos.T), data)
+        return x
+
+    def _full_slice(self, slice_tuple: slice
+                    | tuple[slice | int | EllipsisType, ...]):
+        ndim = len(self.lu)
+        if self.inner_shape:
+            ndim += len(self.inner_shape)
+
+        if isinstance(slice_tuple, slice):
+            slice_tuple = (
+                slice_tuple, ) + (slice(0, sys.maxsize, 1), ) * (ndim - 1)
+        if slice_tuple is Ellipsis:
+            slice_tuple = (slice(0, sys.maxsize, 1), ) * ndim
+        else:
+            head, tail = (), ()
+            for i, s in enumerate(slice_tuple):
+                if s is Ellipsis:
+                    head = slice_tuple[:i]
+                    tail = slice_tuple[i + 1:]
+                    break
+            else:
+                head = slice_tuple
+                tail = ()
+            slice_tuple = head + (slice(
+                0, sys.maxsize, 1), ) * (ndim - len(head) - len(tail)) + tail
+        slice_list = []
+        contract = []
+        reversed = []
+        for i, s in enumerate(slice_tuple):
+            if isinstance(s, int):
+                if s >= 0:
+                    slice_list.append(slice(s, s + 1, 1))
+                elif i < len(self.lu):
+                    s = self.rd[i] + s
+                    slice_list.append(slice(s, s + 1, 1))
+                else:
+                    slice_list.append(slice(s, s - 1, -1))
+                contract.append(i)
+            else:
+                start, stop, step = s.start, s.stop, s.step
+                if step is None:
+                    step = 1
+                if step < 0 and i < len(self.lu):
+                    step = -step
+                    reversed.append(i)
+                    if start is None and stop is None:
+                        start, stop = 0, sys.maxsize
+                    elif start is None:
+                        start, stop = self.lu[i], sys.maxsize
+                    elif stop is None:
+                        start, stop = 0, start + self.lu[i]
+                    else:
+                        start, stop = stop + self.lu[i] + 1, start + self.lu[
+                            i] + 1
+
+                if start is None:
+                    start = 0
+                elif start < 0 and i < len(self.lu):
+                    start = self.rd[i] + start
+                if step is None:
+                    step = 1
+                if stop is None:
+                    stop = sys.maxsize
+                elif stop < 0 and i < len(self.lu):
+                    stop = self.rd[i] + stop
+
+                slice_list.append(slice(start, stop, step))
+        return tuple(slice_list), contract, reversed
+
+    def __getitem__(self, slice_tuple: slice | EllipsisType
+                    | tuple[slice | int | EllipsisType, ...]):
+        self._slice, contract, reversed = self._full_slice(slice_tuple)
+        ret = self.array()
+        slices = []
+        for i, s in enumerate(self._slice):
+            if i in contract:
+                slices.append(0)
+            elif isinstance(s, slice):
+                if i in reversed:
+                    slices.append(slice(None, None, -1))
+                else:
+                    slices.append(slice(None, None, 1))
+        ret = ret.__getitem__(tuple(slices))
+        self._slice = None
+        return ret
+
+
+class Record():
+
+    def __init__(self, id, database, description=None):
+        self.id = id
+        self.database = database
+        self.description = description
+        self._items = {}
+        self._pos = []
+        self._last_vars = set()
+        self._file = None
+
+        if self.is_local_record():
+            self.database = Path(self.database)
+            self._file = random_path(self.database / 'objects')
+            self._file.parent.mkdir(parents=True, exist_ok=True)
+
+    def __getstate__(self) -> dict:
+        return {
+            'id': self.id,
+            'description': self.description,
+            '_items': self._items,
+        }
+
+    def __setstate__(self, state: dict):
+        self.id = state['id']
+        self.description = state['description']
+        self._items = state['_items']
+        self._pos = []
+        self._last_vars = set()
+        self.database = None
+        self._file = None
+
+    @property
+    def axis(self):
+        return self.description.get('axis', {})
+
+    def is_local_record(self):
+        return not self.is_cache_record() and not self.is_remote_record()
+
+    def is_cache_record(self):
+        return self.database is None
+
+    def is_remote_record(self):
+        return isinstance(self.database,
+                          str) and self.database.startswith("tcp://")
+
+    def __del__(self):
+        self.flush()
+
+    def __getitem__(self, key):
+        ret = self.get(key, buffer_to_array=True)
+        if isinstance(ret, Space):
+            ret = ret.toarray()
+        return ret
+
+    def get(self, key, default=_notgiven, buffer_to_array=False, slice=None):
+        if self.is_remote_record():
+            with ZMQContextManager(zmq.DEALER,
+                                   connect=self.database) as socket:
+                socket.send_pyobj({
+                    'method': 'record_getitem',
+                    'record_id': self.id,
+                    'key': key
+                })
+                ret = socket.recv_pyobj()
+                if isinstance(ret, BufferList):
+                    if buffer_to_array:
+                        socket.send_pyobj({
+                            'method': 'bufferlist_slice',
+                            'record_id': self.id,
+                            'key': key,
+                            'slice': slice
+                        })
+                        lst = socket.recv_pyobj()
+                        ret._list = lst
+                        ret._slice = slice
+                        return ret.array()
+                    else:
+                        ret._data_id = self.database, self.id, key
+                        return ret
+                else:
+                    return ret
+        else:
+            if default is _notgiven:
+                d = self._items.get(key)
+            else:
+                d = self._items.get(key, default)
+            if isinstance(d, BufferList):
+                if isinstance(d.file, str):
+                    d.file = self._file.parent.parent.parent.parent / d.file
+                d._slice = slice
+                if buffer_to_array:
+                    return d.array()
+                else:
+                    return d
+            else:
+                return d
+
+    def keys(self):
+        if self.is_remote_record():
+            with ZMQContextManager(zmq.DEALER,
+                                   connect=self.database) as socket:
+                socket.send_pyobj({
+                    'method': 'record_keys',
+                    'record_id': self.id
+                })
+                return socket.recv_pyobj()
+        else:
+            return list(self._items.keys())
+
+    def append(self, level, step, position, variables):
+        if level < 0:
+            self.flush()
+            return
+
+        for key in set(variables.keys()) - self._last_vars:
+            if key not in self.axis:
+                self.axis[key] = tuple(range(level + 1))
+
+        self._last_vars = set(variables.keys())
+
+        if level >= len(self._pos):
+            l = level + 1 - len(self._pos)
+            self._pos.extend(([0] * (l - 1)) + [position])
+            pos = tuple(self._pos)
+        elif level == len(self._pos) - 1:
+            self._pos[-1] = position
+            pos = tuple(self._pos)
+        else:
+            self._pos = self._pos[:level + 1]
+            self._pos[-1] = position
+            pos = tuple(self._pos)
+            self._pos[-1] += 1
+
+        for key, value in variables.items():
+            if self.axis[key] == ():
+                if key not in self._items:
+                    self._items[key] = value
+            elif level == self.axis[key][-1]:
+                if key not in self._items:
+                    if self.is_local_record():
+                        bufferlist_file = random_path(self.database /
+                                                      'objects')
+                        bufferlist_file.parent.mkdir(parents=True,
+                                                     exist_ok=True)
+                        self._items[key] = BufferList(bufferlist_file)
+                    else:
+                        self._items[key] = BufferList()
+                    self._items[key].lu = pos
+                    self._items[key].rd = tuple([i + 1 for i in pos])
+                    self._items[key].append(pos, value, self.axis[key])
+                elif isinstance(self._items[key], BufferList):
+                    self._items[key].append(pos, value, self.axis[key])
+
+    def flush(self):
+        if self.is_remote_record() or self.is_cache_record():
+            return
+
+        for key, value in self._items.items():
+            if isinstance(value, BufferList):
+                value.flush()
+
+        with open(self._file, 'wb') as f:
+            dill.dump(self, f)
+
+    def __repr__(self):
+        return f"<Record: id={self.id} app={self.description['app']}, keys={self.keys()}>"
+
+    # def _repr_html_(self):
+    #     return f"""
+    #     <h3>Record: id={self.id}, app={self.description['app']}</h3>
+    #     <p>keys={self.keys()}</p>
+    #     <p>axis={self.axis}</p>
+    #     """