QuLab 2.0.2__cp311-cp311-macosx_10_9_universal2.whl → 2.0.4__cp311-cp311-macosx_10_9_universal2.whl

{QuLab-2.0.2.dist-info → QuLab-2.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.0.2
+Version: 2.0.4
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>
@@ -32,6 +32,7 @@ Requires-Dist: ipython >=7.4.0
 Requires-Dist: ipywidgets >=7.4.2
 Requires-Dist: loguru >=0.7.2
 Requires-Dist: matplotlib >=3.7.2
+Requires-Dist: nevergrad >=1.0.2
 Requires-Dist: numpy >=1.13.3
 Requires-Dist: ply >=3.11
 Requires-Dist: pyzmq >=25.1.0

{QuLab-2.0.2.dist-info → QuLab-2.0.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qulab/__init__.py,sha256=8zLGg-DfQhnDl2Ky0n-zXpN-8e-g7iR0AcaI4l4Vvpk,32
-qulab/__main__.py,sha256=h84t4vjTH6Gu7SrBhudkzvbqfH1oNudDtM11LIY1h4Q,430
-qulab/fun.cpython-311-darwin.so,sha256=vZyZMkbS3DVPFrFumYWuPDPi3hZbXUQhiri3D_3OxyA,159616
-qulab/version.py,sha256=xARS6PXiulA1Bog5UZUM6hqQfydeRzKLyWtOyUBL-Ss,21
+qulab/__main__.py,sha256=eupSsrNVfnTFRpjgrY_knPvZIs0-Dk577LaN7qB15hI,487
+qulab/fun.cpython-311-darwin.so,sha256=Ts0LNKfAATzvuQVYqCs5zehDzRYyYEXNHT824hdGaGA,159616
+qulab/version.py,sha256=vd1KwOWsA9E1XbsJopSk63EogXGb_qCx8XG7lzXJd8Y,21
 qulab/monitor/__init__.py,sha256=nTHelnDpxRS_fl_B38TsN0njgq8eVTEz9IAnN3NbDlM,42
 qulab/monitor/__main__.py,sha256=w3yUcqq195LzSnXTkQcuC1RSFRhy4oQ_PEBmucXguME,97
 qulab/monitor/config.py,sha256=fQ5JcsMApKc1UwANEnIvbDQZl8uYW0tle92SaYtX9lI,744
@@ -17,10 +17,11 @@ qulab/scan/curd.py,sha256=ntpK62ArZiF2mrDDewcw227VMR1E_8no0yLJSrgdgng,4518
 qulab/scan/expression.py,sha256=-aTYbjFQNI1mwOcoSBztqhKfGJpu_n4a1QnWro_xnTU,15694
 qulab/scan/models.py,sha256=S8Q9hC8nOzxyoNB10EYg-miDKqoNMnjyAECjD-TuORw,17117
 qulab/scan/optimize.py,sha256=vErjRTCtn2MwMF5Xyhs1P4gHF2IFHv_EqxsUvH_4y7k,2287
-qulab/scan/query_record.py,sha256=ed40efBQxtkwUxZHT0zB9SYlMxgNUFqOtCiseOCeBe0,11521
-qulab/scan/recorder.py,sha256=jIkFY-Mirvkpyvh-8nKFiTqrLbGAgp9h9kBX25q_RIs,15402
-qulab/scan/scan.py,sha256=hWKDSosn7_eOAiiA548vdBRYu34OvvWPEtfXLk7fxcY,23030
-qulab/scan/utils.py,sha256=2bj7ggTNTK5LXwBDi8w7IzFqFjchoE6EhgSASC0fpDQ,1024
+qulab/scan/query_record.py,sha256=BVkNgv3yfbMXX_Kguq18fvowKOBmBiiTvUwj_CqpiF4,11493
+qulab/scan/recorder.py,sha256=3-9iqYIg11Ne3vJ2Q8uk9Hp5x6elKBLlfh5EwslkgBA,17611
+qulab/scan/scan.py,sha256=a07dEFJE2JIVRFuySIuINYYdKzIyjbTcnO-i9oaC6_8,25598
+qulab/scan/server.py,sha256=W8z3vr0cqSSKWzIG6_b0d-lpBpDXGpHSwN6VJuv3w9U,2844
+qulab/scan/utils.py,sha256=n5yquKlz2QYMzciPgD9vkpBJVgzVzOqAlfvB4Qu6oOk,2551
 qulab/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qulab/storage/__main__.py,sha256=3emxxRry8BB0m8hUZvJ_oBqkPy7ksV7flHB_KEDXZuI,1692
 qulab/storage/base_dataset.py,sha256=4aKhqBNdZfdlm_z1Qy5dv0HrvgpvMdy8pbyfua8uE-4,11865
@@ -76,9 +77,9 @@ qulab/visualization/plot_layout.py,sha256=clNw9QjE_kVNpIIx2Ob4YhAz2fucPGMuzkoIrO
 qulab/visualization/plot_seq.py,sha256=lphYF4VhkEdc_wWr1kFBwrx2yujkyFPFaJ3pjr61awI,2693
 qulab/visualization/qdat.py,sha256=ZeevBYWkzbww4xZnsjHhw7wRorJCBzbG0iEu-XQB4EA,5735
 qulab/visualization/widgets.py,sha256=6KkiTyQ8J-ei70LbPQZAK35wjktY47w2IveOa682ftA,3180
-QuLab-2.0.2.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
-QuLab-2.0.2.dist-info/METADATA,sha256=cIK74-RBs-jNnttEcwVdgMtNd4WAh7OmcVn2ShZBIt4,3477
-QuLab-2.0.2.dist-info/WHEEL,sha256=eupBwbXGAhwNAPJSvj5BiShZwdZO8jnQ5yHfv-9aUGw,115
-QuLab-2.0.2.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
-QuLab-2.0.2.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
-QuLab-2.0.2.dist-info/RECORD,,
+QuLab-2.0.4.dist-info/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
+QuLab-2.0.4.dist-info/METADATA,sha256=ITxP9PCKiTxbultRyfTgA92HFzX5jch7pWrHZSVyCgE,3510
+QuLab-2.0.4.dist-info/WHEEL,sha256=eupBwbXGAhwNAPJSvj5BiShZwdZO8jnQ5yHfv-9aUGw,115
+QuLab-2.0.4.dist-info/entry_points.txt,sha256=ohBzutEnQimP_BZWiuXdSliu4QAYSHHcN0PZD8c7ZCY,46
+QuLab-2.0.4.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
+QuLab-2.0.4.dist-info/RECORD,,

qulab/__main__.py

@@ -2,6 +2,7 @@ import click
 
 from .monitor.__main__ import main as monitor
 from .scan.recorder import record
+from .scan.server import server
 from .sys.net.cli import dht
 from .visualization.__main__ import plot
 
@@ -21,6 +22,7 @@ main.add_command(monitor)
 main.add_command(plot)
 main.add_command(dht)
 main.add_command(record)
+main.add_command(server)
 
 if __name__ == '__main__':
     main()

qulab/scan/query_record.py

@@ -20,7 +20,6 @@ def get_record(id, database='tcp://127.0.0.1:6789'):
                 'record_id': id
             })
             d = dill.loads(socket.recv_pyobj())
-            print(d.keys())
             return Record(id, database, d)
     else:
         from .models import Record as RecordInDB

qulab/scan/recorder.py

@@ -1,9 +1,12 @@
 import asyncio
+import os
 import pickle
 import sys
 import time
 import uuid
+from collections import defaultdict
 from pathlib import Path
+from threading import Lock
 
 import click
 import dill
@@ -18,7 +21,16 @@ from .models import Record as RecordInDB
 from .models import Session, create_engine, create_tables, sessionmaker, utcnow
 
 _notgiven = object()
-datapath = Path.home() / 'qulab' / 'data'
+
+try:
+    default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
+except:
+    default_record_port = 6789
+
+if os.getenv('QULAB_RECORD_PATH'):
+    datapath = Path(os.getenv('QULAB_RECORD_PATH'))
+else:
+    datapath = Path.home() / 'qulab' / 'data'
 datapath.mkdir(parents=True, exist_ok=True)
 
 record_cache = {}
@@ -41,24 +53,49 @@ class BufferList():
         self.rd = ()
         self.pos_file = pos_file
         self.value_file = value_file
+        self._lock = Lock()
+
+    def __getstate__(self):
+        return {
+            'pos_file': self.pos_file,
+            'value_file': self.value_file,
+            '_pos': self._pos,
+            '_value': self._value,
+            'lu': self.lu,
+            'rd': self.rd
+        }
+
+    def __setstate__(self, state):
+        self.pos_file = state['pos_file']
+        self.value_file = state['value_file']
+        self._pos = state['_pos']
+        self._value = state['_value']
+        self.lu = state['lu']
+        self.rd = state['rd']
+        self._lock = Lock()
 
     @property
     def shape(self):
         return tuple([i - j for i, j in zip(self.rd, self.lu)])
 
     def flush(self):
-        if self.pos_file is not None:
-            with open(self.pos_file, 'ab') as f:
-                for pos in self._pos:
-                    dill.dump(pos, f)
-            self._pos.clear()
-        if self.value_file is not None:
-            with open(self.value_file, 'ab') as f:
-                for value in self._value:
-                    dill.dump(value, f)
-            self._value.clear()
-
-    def append(self, pos, value):
+        with self._lock:
+            if self.pos_file is not None:
+                with open(self.pos_file, 'ab') as f:
+                    for pos in self._pos:
+                        dill.dump(pos, f)
+                self._pos.clear()
+            if self.value_file is not None:
+                with open(self.value_file, 'ab') as f:
+                    for value in self._value:
+                        dill.dump(value, f)
+                self._value.clear()
+
+    def append(self, pos, value, dims=None):
+        if dims is not None:
+            if any([p != 0 for i, p in enumerate(pos) if i not in dims]):
+                return
+            pos = tuple([pos[i] for i in dims])
         self.lu = tuple([min(i, j) for i, j in zip(pos, self.lu)])
         self.rd = tuple([max(i + 1, j) for i, j in zip(pos, self.rd)])
         self._pos.append(pos)
@@ -68,25 +105,27 @@ class BufferList():
 
     def value(self):
         v = []
-        if self.value_file is not None:
-            with open(self.value_file, 'rb') as f:
-                while True:
-                    try:
-                        v.append(dill.load(f))
-                    except EOFError:
-                        break
+        if self.value_file is not None and self.value_file.exists():
+            with self._lock:
+                with open(self.value_file, 'rb') as f:
+                    while True:
+                        try:
+                            v.append(dill.load(f))
+                        except EOFError:
+                            break
         v.extend(self._value)
         return v
 
     def pos(self):
         p = []
-        if self.pos_file is not None:
-            with open(self.pos_file, 'rb') as f:
-                while True:
-                    try:
-                        p.append(dill.load(f))
-                    except EOFError:
-                        break
+        if self.pos_file is not None and self.pos_file.exists():
+            with self._lock:
+                with open(self.pos_file, 'rb') as f:
+                    while True:
+                        try:
+                            p.append(dill.load(f))
+                        except EOFError:
+                            break
         p.extend(self._pos)
         return p
 
@@ -114,21 +153,32 @@ class Record():
         self._file = None
         self.independent_variables = {}
         self.constants = {}
-
-        for level, group in self.description['order'].items():
-            for names in group:
-                for name in names:
-                    self._levels[name] = level
+        self.dims = {}
 
         for name, value in self.description['consts'].items():
             if name not in self._items:
                 self._items[name] = value
             self.constants[name] = value
+            self.dims[name] = ()
         for level, range_list in self.description['loops'].items():
             for name, iterable in range_list:
                 if isinstance(iterable, (np.ndarray, list, tuple, range)):
                     self._items[name] = iterable
-                    self.independent_variables[name] = (level, iterable)
+                    self.independent_variables[name] = iterable
+                    self.dims[name] = (level, )
+
+        for level, group in self.description['order'].items():
+            for names in group:
+                for name in names:
+                    self._levels[name] = level
+                    if name not in self.dims:
+                        if name not in self.description['dependents']:
+                            self.dims[name] = (level, )
+                        else:
+                            d = set()
+                            for n in self.description['dependents'][name]:
+                                d.update(self.dims[n])
+                            self.dims[name] = tuple(sorted(d))
 
         if self.is_local_record():
             self.database = Path(self.database)
@@ -203,6 +253,7 @@ class Record():
         for key in set(variables.keys()) - self._last_vars:
             if key not in self._levels:
                 self._levels[key] = level
+                self.dims[key] = tuple(range(level + 1))
 
         self._last_vars = set(variables.keys())
         self._keys.update(variables.keys())
@@ -237,9 +288,9 @@ class Record():
                         self._items[key] = BufferList()
                     self._items[key].lu = pos
                     self._items[key].rd = tuple([i + 1 for i in pos])
-                    self._items[key].append(pos, value)
+                    self._items[key].append(pos, value, self.dims[key])
                 elif isinstance(self._items[key], BufferList):
-                    self._items[key].append(pos, value)
+                    self._items[key].append(pos, value, self.dims[key])
             elif self._levels[key] == -1 and key not in self._items:
                 self._items[key] = value
 
@@ -283,7 +334,7 @@ def flush_cache():
         r.flush()
 
 
-def get_record(session, id, datapath):
+def get_record(session: Session, id: int, datapath: Path) -> Record:
     if id not in record_cache:
         record_in_db = session.get(RecordInDB, id)
         record_in_db.atime = utcnow()
@@ -297,7 +348,7 @@ def get_record(session, id, datapath):
     return record
 
 
-def create_record(session, description, datapath):
+def record_create(session: Session, description: dict, datapath: Path) -> int:
     record = Record(None, datapath, description)
     record_in_db = RecordInDB()
     if 'app' in description:
@@ -317,6 +368,20 @@ def create_record(session, description, datapath):
         raise
 
 
+def record_append(session: Session, record_id: int, level: int, step: int,
+                  position: int, variables: dict, datapath: Path):
+    record = get_record(session, record_id, datapath)
+    record.append(level, step, position, variables)
+    try:
+        record_in_db = session.get(RecordInDB, record_id)
+        record_in_db.mtime = utcnow()
+        record_in_db.atime = utcnow()
+        session.commit()
+    except:
+        session.rollback()
+        raise
+
+
 @logger.catch
 async def handle(session: Session, request: Request, datapath: Path):
 
@@ -327,16 +392,10 @@ async def handle(session: Session, request: Request, datapath: Path):
             await reply(request, 'pong')
         case 'record_create':
             description = dill.loads(msg['description'])
-            await reply(request, create_record(session, description, datapath))
+            await reply(request, record_create(session, description, datapath))
         case 'record_append':
-            record = get_record(session, msg['record_id'], datapath)
-            record.append(msg['level'], msg['step'], msg['position'],
-                          msg['variables'])
-            if msg['level'] < 0:
-                record_in_db = session.get(RecordInDB, msg['record_id'])
-                record_in_db.mtime = utcnow()
-                record_in_db.atime = utcnow()
-                session.commit()
+            record_append(session, msg['record_id'], msg['level'], msg['step'],
+                          msg['position'], msg['variables'], datapath)
         case 'record_description':
             record = get_record(session, msg['record_id'], datapath)
             await reply(request, dill.dumps(record.description))
@@ -365,7 +424,7 @@ async def handle(session: Session, request: Request, datapath: Path):
         case 'record_replace_tags':
             update_tags(session, msg['record_id'], msg['tags'], False)
         case _:
-            logger.error(f'Unknown method: {msg["method"]}')
+            logger.error(f"Unknown method: {msg['method']}")
 
 
 async def _handle(session: Session, request: Request, datapath: Path):
@@ -431,7 +490,9 @@ async def main(port, datapath, url, timeout=1, buffer=1024, interval=60):
 
 
 @click.command()
-@click.option('--port', default=6789, help='Port of the server.')
+@click.option('--port',
+              default=os.getenv('QULAB_RECORD_PORT', 6789),
+              help='Port of the server.')
 @click.option('--datapath', default=datapath, help='Path of the data.')
 @click.option('--url', default=None, help='URL of the database.')
 @click.option('--timeout', default=1, help='Timeout of ping.')